From 3c1a83e222240307a21310ddb0b2bb6d0559d23a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 3 Aug 2021 12:05:53 +0200 Subject: [PATCH 001/103] Outsource cmd/icingadb.Exit* to cmd/internal --- cmd/icingadb/main.go | 7 +++---- cmd/internal/common.go | 6 ++++++ 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 cmd/internal/common.go diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go index d778c412..fb8bc7d9 100644 --- a/cmd/icingadb/main.go +++ b/cmd/icingadb/main.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/go-redis/redis/v8" + "github.com/icinga/icingadb/cmd/internal" "github.com/icinga/icingadb/internal/command" "github.com/icinga/icingadb/pkg/common" "github.com/icinga/icingadb/pkg/driver" @@ -29,8 +30,6 @@ import ( ) const ( - ExitSuccess = 0 - ExitFailure = 1 expectedRedisSchemaVersion = "5" expectedMysqlSchemaVersion = 3 expectedPostgresSchemaVersion = 1 @@ -343,14 +342,14 @@ func run() int { } cancelHactx() - return ExitFailure + return internal.ExitFailure case <-ctx.Done(): logger.Fatalf("%+v", errors.New("main context closed unexpectedly")) case s := <-sig: logger.Infow("Exiting due to signal", zap.String("signal", s.String())) cancelHactx() - return ExitSuccess + return internal.ExitSuccess } } diff --git a/cmd/internal/common.go b/cmd/internal/common.go new file mode 100644 index 00000000..0aa96924 --- /dev/null +++ b/cmd/internal/common.go @@ -0,0 +1,6 @@ +package internal + +const ( + ExitSuccess = 0 + ExitFailure = 1 +) From 700f88a2ffa13cd6b7521bde9ac6a95987a55906 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 3 Aug 2021 12:58:03 +0200 Subject: [PATCH 002/103] cmd/ido2icingadb: parse CLI flags --- cmd/ido2icingadb/main.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 cmd/ido2icingadb/main.go diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go new file mode 100644 index 00000000..463a0274 --- /dev/null +++ b/cmd/ido2icingadb/main.go @@ -0,0 +1,27 @@ +package main + +import ( + "github.com/icinga/icingadb/cmd/internal" + "github.com/jessevdk/go-flags" + "os" +) + +// Flags defines the CLI flags. +type Flags struct { + // Config is the path to the config file. + Config string `short:"c" long:"config" description:"path to config file" required:"true"` +} + +func main() { + os.Exit(run()) +} + +func run() int { + f := &Flags{} + if _, err := flags.NewParser(f, flags.Default).Parse(); err != nil { + return 2 + } + + // TODO + return internal.ExitSuccess +} From dce3889645b7413bba30d3163800978265edf63c Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 3 Aug 2021 13:30:41 +0200 Subject: [PATCH 003/103] cmd/ido2icingadb: parse config file --- cmd/ido2icingadb/main.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 463a0274..99340934 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -1,7 +1,10 @@ package main import ( + "fmt" + "github.com/goccy/go-yaml" "github.com/icinga/icingadb/cmd/internal" + "github.com/icinga/icingadb/pkg/config" "github.com/jessevdk/go-flags" "os" ) @@ -12,6 +15,12 @@ type Flags struct { Config string `short:"c" long:"config" description:"path to config file" required:"true"` } +// Config defines the YAML config structure. +type Config struct { + IDO config.Database `yaml:"ido"` + IcingaDB config.Database `yaml:"icingadb"` +} + func main() { os.Exit(run()) } @@ -22,6 +31,24 @@ func run() int { return 2 } + cf, err := os.Open(f.Config) + if err != nil { + fmt.Fprintf(os.Stderr, "can't open config file: %s\n", err.Error()) + return 2 + } + + c := &Config{} + + { + err := yaml.NewDecoder(cf).Decode(c) + cf.Close() + + if err != nil { + fmt.Fprintf(os.Stderr, "can't parse config file: %s\n", err.Error()) + return 2 + } + } + // TODO return internal.ExitSuccess } From 890b66ddb9f9deb3e7a28d45274ee05abfd02fd9 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 3 Aug 2021 18:54:29 +0200 Subject: [PATCH 004/103] cmd/ido2icingadb: connect to databases --- cmd/ido2icingadb/main.go | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 99340934..82942b3c 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -1,11 +1,16 @@ package main import ( + "context" "fmt" "github.com/goccy/go-yaml" "github.com/icinga/icingadb/cmd/internal" "github.com/icinga/icingadb/pkg/config" + "github.com/icinga/icingadb/pkg/icingadb" "github.com/jessevdk/go-flags" + "github.com/pkg/errors" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" "os" ) @@ -49,6 +54,48 @@ func run() int { } } + logger, _ := zap.NewDevelopmentConfig().Build() + + log := logger.Sugar() + defer log.Sync() + + log.Info("Starting IDO to Icinga DB history migration") + + log.Info("Connecting to databases") + var ido, idb *icingadb.DB + + { + eg, _ := errgroup.WithContext(context.Background()) + + eg.Go(func() error { + ido = connect(log, "IDO", &c.IDO) + return nil + }) + + eg.Go(func() error { + idb = connect(log, "Icinga DB", &c.IcingaDB) + return nil + }) + + _ = eg.Wait() + } + // TODO + _ = ido + _ = idb + return internal.ExitSuccess } + +func connect(log *zap.SugaredLogger, which string, cfg *config.Database) *icingadb.DB { + db, err := cfg.Open(log) + if err != nil { + log.With("backend", which).Fatalf("%+v", errors.Wrap(err, "can't connect to database")) + } + + if err := db.Ping(); err != nil { + log.With("backend", which).Fatalf("%+v", errors.Wrap(err, "can't connect to database")) + } + + return db +} From 4eb2ba1ea4aa595be281364b2fd246a95eb2684d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 3 Aug 2021 19:08:54 +0200 Subject: [PATCH 005/103] cmd/ido2icingadb: count total IDO events --- cmd/ido2icingadb/main.go | 59 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 82942b3c..fd9d971d 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -2,12 +2,14 @@ package main import ( "context" + "database/sql" "fmt" "github.com/goccy/go-yaml" "github.com/icinga/icingadb/cmd/internal" "github.com/icinga/icingadb/pkg/config" "github.com/icinga/icingadb/pkg/icingadb" "github.com/jessevdk/go-flags" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -80,8 +82,63 @@ func run() int { _ = eg.Wait() } + var types = []struct { + name string + idoTable string + snapshot *sqlx.Tx + }{ + {"acknowledgement", "icinga_acknowledgements", nil}, {"comment", "icinga_commenthistory", nil}, + {"downtime", "icinga_downtimehistory", nil}, {"flapping", "icinga_flappinghistory", nil}, + {"notification", "icinga_notifications", nil}, {"state", "icinga_statehistory", nil}, + } + + { + eg, _ := errgroup.WithContext(context.Background()) + for i := range types { + i := i + + eg.Go(func() error { + tx, err := ido.BeginTxx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't begin snapshot transaction")) + } + + types[i].snapshot = tx + return nil + }) + } + + _ = eg.Wait() + } + + log.Info("Computing progress") + + { + eg, _ := errgroup.WithContext(context.Background()) + for i := range types { + i := i + + eg.Go(func() error { + var count uint64 + + err := types[i].snapshot.Get( + &count, + "SELECT COUNT(*) FROM "+types[i].idoTable+ + " xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id", + ) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't count query")) + } + + log.With("type", types[i].name, "amount", count).Info("Counted total IDO events") + return nil + }) + } + + _ = eg.Wait() + } + // TODO - _ = ido _ = idb return internal.ExitSuccess From 05c736aa8f6cb4fa75850716ac4036385eb582fa Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 7 Dec 2021 16:52:52 +0100 Subject: [PATCH 006/103] .../compliance/check-licenses.sh: also check UNLICENSE --- .github/workflows/compliance/check-licenses.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compliance/check-licenses.sh b/.github/workflows/compliance/check-licenses.sh index 37d35b8f..93c174a4 100755 --- a/.github/workflows/compliance/check-licenses.sh +++ b/.github/workflows/compliance/check-licenses.sh @@ -5,7 +5,7 @@ set -eo pipefail find_license_file() { MOD_NAME="$1" LICENSE_DIR="vendor/$MOD_NAME" - LICENSE_FILES=({,../}LICENSE{,.txt,.md}) + LICENSE_FILES=({,../}{,UN}LICENSE{,.txt,.md}) for LICENSE_FILE in "${LICENSE_FILES[@]}"; do LICENSE_FILE="${LICENSE_DIR}/$LICENSE_FILE" From 2f5d5ea931fb20665f4d7bbf07e16220947a10d0 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 20 Aug 2021 18:45:59 +0200 Subject: [PATCH 007/103] cmd/ido2icingadb: compute previous progress --- .../workflows/compliance/check-licenses.sh | 2 + cmd/ido2icingadb/main.go | 161 ++++++++++++++++-- cmd/ido2icingadb/misc.go | 149 ++++++++++++++++ cmd/ido2icingadb/misc_test.go | 15 ++ go.mod | 5 + go.sum | 12 ++ tests/go.sum | 8 + 7 files changed, 336 insertions(+), 16 deletions(-) create mode 100644 cmd/ido2icingadb/misc.go create mode 100644 cmd/ido2icingadb/misc_test.go diff --git a/.github/workflows/compliance/check-licenses.sh b/.github/workflows/compliance/check-licenses.sh index 93c174a4..63ff76f6 100755 --- a/.github/workflows/compliance/check-licenses.sh +++ b/.github/workflows/compliance/check-licenses.sh @@ -29,6 +29,8 @@ list_all_deps() { COMPATIBLE_LINE=$(($LINENO + 2)) COMPATIBLE=( + # public domain + 3cee2c43614ad4572d9d594c81b9348cf45ed5ac # vendor/github.com/vbauerster/mpb/v6/UNLICENSE # MIT 66d504eb2f162b9cbf11b07506eeed90c6edabe1 # vendor/github.com/cespare/xxhash/v2/LICENSE.txt 1513ff663e946fdcadb630bed670d253b8b22e1e # vendor/github.com/davecgh/go-spew/spew/../LICENSE diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index fd9d971d..51c73d94 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "database/sql" "fmt" @@ -11,9 +12,13 @@ import ( "github.com/jessevdk/go-flags" "github.com/jmoiron/sqlx" "github.com/pkg/errors" + "github.com/vbauerster/mpb/v6" + "github.com/vbauerster/mpb/v6/decor" "go.uber.org/zap" "golang.org/x/sync/errgroup" "os" + "strings" + "time" ) // Flags defines the CLI flags. @@ -26,6 +31,9 @@ type Flags struct { type Config struct { IDO config.Database `yaml:"ido"` IcingaDB config.Database `yaml:"icingadb"` + Icinga2 struct { + Env string `yaml:"env"` + } `yaml:"icinga2"` } func main() { @@ -82,16 +90,6 @@ func run() int { _ = eg.Wait() } - var types = []struct { - name string - idoTable string - snapshot *sqlx.Tx - }{ - {"acknowledgement", "icinga_acknowledgements", nil}, {"comment", "icinga_commenthistory", nil}, - {"downtime", "icinga_downtimehistory", nil}, {"flapping", "icinga_flappinghistory", nil}, - {"notification", "icinga_notifications", nil}, {"state", "icinga_statehistory", nil}, - } - { eg, _ := errgroup.WithContext(context.Background()) for i := range types { @@ -119,10 +117,8 @@ func run() int { i := i eg.Go(func() error { - var count uint64 - err := types[i].snapshot.Get( - &count, + &types[i].total, "SELECT COUNT(*) FROM "+types[i].idoTable+ " xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id", ) @@ -130,7 +126,7 @@ func run() int { log.Fatalf("%+v", errors.Wrap(err, "can't count query")) } - log.With("type", types[i].name, "amount", count).Info("Counted total IDO events") + log.With("type", types[i].name, "amount", types[i].total).Info("Counted total IDO events") return nil }) } @@ -138,8 +134,141 @@ func run() int { _ = eg.Wait() } - // TODO - _ = idb + log.Sync() + + { + progress := mpb.New() + for i := range types { + typ := &types[i] + + typ.bar = progress.AddBar( + typ.total, + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators( + decor.Name(typ.name, decor.WC{W: len(typ.name) + 1, C: decor.DidentRight}), + decor.Percentage(decor.WC{W: 5}), + ), + mpb.AppendDecorators(decor.EwmaETA(decor.ET_STYLE_GO, 6000000, decor.WC{W: 4})), + ) + } + + eg, _ := errgroup.WithContext(context.Background()) + for i := range types { + typ := &types[i] + + if typ.total == 0 { + typ.bar.SetTotal(typ.bar.Current(), true) + } else { + eg.Go(func() error { + query := "SELECT xh." + + strings.Join(append(append([]string(nil), typ.idoColumns...), typ.idoIdColumn), ", xh.") + + " id FROM " + typ.idoTable + + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + + typ.idoIdColumn + " > ? ORDER BY xh." + typ.idoIdColumn + " LIMIT 10000" + + stmt, err := typ.snapshot.Preparex(query) + if err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + defer stmt.Close() + + var lastRowsLen int + var lastQuery string + var lastStmt *sqlx.Stmt + start := time.Now() + + defer func() { + if lastStmt != nil { + lastStmt.Close() + } + }() + + Queries: + for { + var rows []ProgressRow + if err := stmt.Select(&rows, typ.lastId); err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + + if len(rows) < 1 { + break + } + + if len(rows) != lastRowsLen { + if lastStmt != nil { + lastStmt.Close() + } + + buf := &bytes.Buffer{} + fmt.Fprintf( + buf, "SELECT %s FROM %s WHERE %s IN (?", typ.idbIdColumn, typ.idbTable, typ.idbIdColumn, + ) + + for i := 1; i < len(rows); i++ { + buf.Write([]byte(",?")) + } + + buf.Write([]byte(")")) + lastRowsLen = len(rows) + lastQuery = buf.String() + + var err error + lastStmt, err = idb.Preparex(lastQuery) + + if err != nil { + log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + } + + ids := make([]interface{}, 0, len(rows)) + converted := make([]convertedId, 0, len(rows)) + + for _, row := range rows { + conv := typ.convertId(row, c.Icinga2.Env) + ids = append(ids, conv) + converted = append(converted, convertedId{row.Id, conv}) + } + + var present [][]byte + if err := lastStmt.Select(&present, ids...); err != nil { + log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + + presentSet := map[[20]byte]struct{}{} + for _, row := range present { + var key [20]byte + copy(key[:], row) + presentSet[key] = struct{}{} + } + + for _, conv := range converted { + var key [20]byte + copy(key[:], conv.idb) + + if _, ok := presentSet[key]; !ok { + break Queries + } + + typ.lastId = conv.ido + } + + prev := start + now := time.Now() + start = now + + typ.bar.IncrBy(len(rows)) + typ.bar.DecoratorEwmaUpdate(now.Sub(prev)) + } + + typ.bar.SetTotal(typ.bar.Current(), true) + return nil + }) + } + } + + _ = eg.Wait() + progress.Wait() + } return internal.ExitSuccess } diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go new file mode 100644 index 00000000..8a3e09fd --- /dev/null +++ b/cmd/ido2icingadb/misc.go @@ -0,0 +1,149 @@ +package main + +import ( + "bytes" + "crypto/sha1" + "encoding/binary" + "github.com/google/uuid" + "github.com/icinga/icingadb/pkg/icingadb/objectpacker" + "github.com/jmoiron/sqlx" + "github.com/vbauerster/mpb/v6" +) + +type ProgressRow struct { + Id uint64 + Name string +} + +type convertedId struct { + ido uint64 + idb []byte +} + +// mkDeterministicUuid returns a formally random UUID (v4) as follows: 11111122-3300-4455-4455-555555555555 +// +// 0: zeroed +// 1: "IDO" (where the data identified by the new UUID is from) +// 2: the history table the new UUID is for, e.g. "s" for state_history +// 3: "h" (for "history") +// 4: the new UUID's formal version (unused bits zeroed) +// 5: the ID of the row the new UUID is for in the IDO (big endian) +func mkDeterministicUuid(table byte, rowId uint64) []byte { + uid := uuidTemplate + uid[3] = table + + buf := &bytes.Buffer{} + if err := binary.Write(buf, binary.BigEndian, rowId); err != nil { + panic(err) + } + + bEId := buf.Bytes() + uid[7] = bEId[0] + copy(uid[9:], bEId[1:]) + + return uid[:] +} + +// uuidTemplate is for mkDeterministicUuid. +var uuidTemplate = func() uuid.UUID { + buf := &bytes.Buffer{} + buf.Write(uuid.Nil[:]) + + uid, err := uuid.NewRandomFromReader(buf) + if err != nil { + panic(err) + } + + copy(uid[:], "IDO h") + return uid +}() + +// hashAny combines PackAny and SHA1 hashing. +func hashAny(in interface{}) []byte { + hash := sha1.New() + if err := objectpacker.PackAny(in, hash); err != nil { + panic(err) + } + + return hash.Sum(nil) +} + +// calcObjectId calculates the ID of the config object named name1 for Icinga DB. +func calcObjectId(env, name1 string) []byte { + return hashAny([2]string{env, name1}) +} + +var types = [6]struct { + name string + idoTable string + idoIdColumn string + idoColumns []string + idbTable string + idbIdColumn string + convertId func(row ProgressRow, env string) []byte + snapshot *sqlx.Tx + total int64 + bar *mpb.Bar + lastId uint64 +}{ + { + "acknowledgement", + "icinga_acknowledgements", + "acknowledgement_id", + nil, + "history", + "id", + func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('a', row.Id) }, + nil, 0, nil, 0, + }, + { + "comment", + "icinga_commenthistory", + "commenthistory_id", + []string{"name"}, + "comment_history", + "comment_id", + func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, + nil, 0, nil, 0, + }, + { + "downtime", + "icinga_downtimehistory", + "downtimehistory_id", + []string{"name"}, + "downtime_history", + "downtime_id", + func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, + nil, 0, nil, 0, + }, + { + "flapping", + "icinga_flappinghistory", + "flappinghistory_id", + nil, + "history", + "id", + func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('f', row.Id) }, + nil, 0, nil, 0, + }, + { + "notification", + "icinga_notifications", + "notification_id", + nil, + "notification_history", + "id", + func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('n', row.Id) }, + nil, 0, nil, 0, + }, + { + "state", + "icinga_statehistory", + "statehistory_id", + nil, + "state_history", + "id", + func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('s', row.Id) }, + nil, 0, nil, 0, + }, +} diff --git a/cmd/ido2icingadb/misc_test.go b/cmd/ido2icingadb/misc_test.go new file mode 100644 index 00000000..76c5aec4 --- /dev/null +++ b/cmd/ido2icingadb/misc_test.go @@ -0,0 +1,15 @@ +package main + +import ( + "bytes" + "testing" +) + +func TestMkDeterministicUuid(t *testing.T) { + if !bytes.Equal( + mkDeterministicUuid('s', 0x0102030405060708), + []byte{'I', 'D', 'O', 's', 'h', 0, 0x40, 1, 0x80, 2, 3, 4, 5, 6, 7, 8}, + ) { + t.Error("got wrong UUID from mkDeterministicUuid(stateHistory, 0x0102030405060708)") + } +} diff --git a/go.mod b/go.mod index cec21d7c..a550de01 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,15 @@ require ( github.com/pkg/errors v0.9.1 github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.8.0 + github.com/vbauerster/mpb/v6 v6.0.4 go.uber.org/zap v1.21.0 golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) require ( + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -28,7 +31,9 @@ require ( github.com/fatih/color v1.10.0 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-runewidth v0.0.12 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect diff --git a/go.sum b/go.sum index 593de8b2..4339a855 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -45,6 +49,8 @@ github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -57,6 +63,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/ssgreg/journald v1.0.0 h1:0YmTDPJXxcWDPba12qNMdO6TxvfkFSYpFIJ31CwmLcU= github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4JEP+oRt0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -67,6 +76,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/vbauerster/mpb/v6 v6.0.4 h1:h6J5zM/2wimP5Hj00unQuV8qbo5EPcj6wbkCqgj7KcY= +github.com/vbauerster/mpb/v6 v6.0.4/go.mod h1:a/+JT57gqh6Du0Ay5jSR+uBMfXGdlR7VQlGP52fJxLM= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -100,6 +111,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/tests/go.sum b/tests/go.sum index 190631a7..964ea7fa 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -66,6 +66,8 @@ github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBK github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -465,6 +467,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= @@ -584,6 +587,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -642,6 +647,8 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbauerster/mpb/v6 v6.0.4 h1:h6J5zM/2wimP5Hj00unQuV8qbo5EPcj6wbkCqgj7KcY= +github.com/vbauerster/mpb/v6 v6.0.4/go.mod h1:a/+JT57gqh6Du0Ay5jSR+uBMfXGdlR7VQlGP52fJxLM= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= @@ -857,6 +864,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 495e0afe32e06ca4969e8bcd01c18f9e93a826d8 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 20 Aug 2021 19:00:29 +0200 Subject: [PATCH 008/103] cmd/ido2icingadb: deduplicate parallelism --- cmd/ido2icingadb/main.go | 268 +++++++++++++++++---------------------- cmd/ido2icingadb/misc.go | 23 +++- 2 files changed, 140 insertions(+), 151 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 51c73d94..381a85f7 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -90,49 +90,28 @@ func run() int { _ = eg.Wait() } - { - eg, _ := errgroup.WithContext(context.Background()) - for i := range types { - i := i - - eg.Go(func() error { - tx, err := ido.BeginTxx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) - if err != nil { - log.Fatalf("%+v", errors.Wrap(err, "can't begin snapshot transaction")) - } - - types[i].snapshot = tx - return nil - }) + types.forEach(func(ht *historyType) { + tx, err := ido.BeginTxx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't begin snapshot transaction")) } - _ = eg.Wait() - } + ht.snapshot = tx + }) log.Info("Computing progress") - { - eg, _ := errgroup.WithContext(context.Background()) - for i := range types { - i := i - - eg.Go(func() error { - err := types[i].snapshot.Get( - &types[i].total, - "SELECT COUNT(*) FROM "+types[i].idoTable+ - " xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id", - ) - if err != nil { - log.Fatalf("%+v", errors.Wrap(err, "can't count query")) - } - - log.With("type", types[i].name, "amount", types[i].total).Info("Counted total IDO events") - return nil - }) + types.forEach(func(ht *historyType) { + err := ht.snapshot.Get( + &ht.total, + "SELECT COUNT(*) FROM "+ht.idoTable+" xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id", + ) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't count query")) } - _ = eg.Wait() - } + log.With("type", ht.name, "amount", ht.total).Info("Counted total IDO events") + }) log.Sync() @@ -152,121 +131,112 @@ func run() int { ) } - eg, _ := errgroup.WithContext(context.Background()) - for i := range types { - typ := &types[i] - - if typ.total == 0 { - typ.bar.SetTotal(typ.bar.Current(), true) - } else { - eg.Go(func() error { - query := "SELECT xh." + - strings.Join(append(append([]string(nil), typ.idoColumns...), typ.idoIdColumn), ", xh.") + - " id FROM " + typ.idoTable + - " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + - typ.idoIdColumn + " > ? ORDER BY xh." + typ.idoIdColumn + " LIMIT 10000" - - stmt, err := typ.snapshot.Preparex(query) - if err != nil { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - defer stmt.Close() - - var lastRowsLen int - var lastQuery string - var lastStmt *sqlx.Stmt - start := time.Now() - - defer func() { - if lastStmt != nil { - lastStmt.Close() - } - }() - - Queries: - for { - var rows []ProgressRow - if err := stmt.Select(&rows, typ.lastId); err != nil { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) - } - - if len(rows) < 1 { - break - } - - if len(rows) != lastRowsLen { - if lastStmt != nil { - lastStmt.Close() - } - - buf := &bytes.Buffer{} - fmt.Fprintf( - buf, "SELECT %s FROM %s WHERE %s IN (?", typ.idbIdColumn, typ.idbTable, typ.idbIdColumn, - ) - - for i := 1; i < len(rows); i++ { - buf.Write([]byte(",?")) - } - - buf.Write([]byte(")")) - lastRowsLen = len(rows) - lastQuery = buf.String() - - var err error - lastStmt, err = idb.Preparex(lastQuery) - - if err != nil { - log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - } - - ids := make([]interface{}, 0, len(rows)) - converted := make([]convertedId, 0, len(rows)) - - for _, row := range rows { - conv := typ.convertId(row, c.Icinga2.Env) - ids = append(ids, conv) - converted = append(converted, convertedId{row.Id, conv}) - } - - var present [][]byte - if err := lastStmt.Select(&present, ids...); err != nil { - log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't perform query")) - } - - presentSet := map[[20]byte]struct{}{} - for _, row := range present { - var key [20]byte - copy(key[:], row) - presentSet[key] = struct{}{} - } - - for _, conv := range converted { - var key [20]byte - copy(key[:], conv.idb) - - if _, ok := presentSet[key]; !ok { - break Queries - } - - typ.lastId = conv.ido - } - - prev := start - now := time.Now() - start = now - - typ.bar.IncrBy(len(rows)) - typ.bar.DecoratorEwmaUpdate(now.Sub(prev)) - } - - typ.bar.SetTotal(typ.bar.Current(), true) - return nil - }) + types.forEach(func(ht *historyType) { + if ht.total == 0 { + ht.bar.SetTotal(ht.bar.Current(), true) + return } - } - _ = eg.Wait() + query := "SELECT xh." + + strings.Join(append(append([]string(nil), ht.idoColumns...), ht.idoIdColumn), ", xh.") + " id FROM " + + ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + + ht.idoIdColumn + " > ? ORDER BY xh." + ht.idoIdColumn + " LIMIT 10000" + + stmt, err := ht.snapshot.Preparex(query) + if err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + defer stmt.Close() + + var lastRowsLen int + var lastQuery string + var lastStmt *sqlx.Stmt + start := time.Now() + + defer func() { + if lastStmt != nil { + lastStmt.Close() + } + }() + + Queries: + for { + var rows []ProgressRow + if err := stmt.Select(&rows, ht.lastId); err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + + if len(rows) < 1 { + break + } + + if len(rows) != lastRowsLen { + if lastStmt != nil { + lastStmt.Close() + } + + buf := &bytes.Buffer{} + fmt.Fprintf(buf, "SELECT %s FROM %s WHERE %s IN (?", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn) + + for i := 1; i < len(rows); i++ { + buf.Write([]byte(",?")) + } + + buf.Write([]byte(")")) + lastRowsLen = len(rows) + lastQuery = buf.String() + + var err error + lastStmt, err = idb.Preparex(lastQuery) + + if err != nil { + log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + } + + ids := make([]interface{}, 0, len(rows)) + converted := make([]convertedId, 0, len(rows)) + + for _, row := range rows { + conv := ht.convertId(row, c.Icinga2.Env) + ids = append(ids, conv) + converted = append(converted, convertedId{row.Id, conv}) + } + + var present [][]byte + if err := lastStmt.Select(&present, ids...); err != nil { + log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + + presentSet := map[[20]byte]struct{}{} + for _, row := range present { + var key [20]byte + copy(key[:], row) + presentSet[key] = struct{}{} + } + + for _, conv := range converted { + var key [20]byte + copy(key[:], conv.idb) + + if _, ok := presentSet[key]; !ok { + break Queries + } + + ht.lastId = conv.ido + } + + prev := start + now := time.Now() + start = now + + ht.bar.IncrBy(len(rows)) + ht.bar.DecoratorEwmaUpdate(now.Sub(prev)) + } + + ht.bar.SetTotal(ht.bar.Current(), true) + }) + progress.Wait() } diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 8a3e09fd..f0859779 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -2,12 +2,14 @@ package main import ( "bytes" + "context" "crypto/sha1" "encoding/binary" "github.com/google/uuid" "github.com/icinga/icingadb/pkg/icingadb/objectpacker" "github.com/jmoiron/sqlx" "github.com/vbauerster/mpb/v6" + "golang.org/x/sync/errgroup" ) type ProgressRow struct { @@ -73,7 +75,7 @@ func calcObjectId(env, name1 string) []byte { return hashAny([2]string{env, name1}) } -var types = [6]struct { +type historyType struct { name string idoTable string idoIdColumn string @@ -85,7 +87,24 @@ var types = [6]struct { total int64 bar *mpb.Bar lastId uint64 -}{ +} + +type historyTypes [6]historyType + +func (ht *historyTypes) forEach(f func(*historyType)) { + eg, _ := errgroup.WithContext(context.Background()) + for i := range *ht { + i := i + eg.Go(func() error { + f(&(*ht)[i]) + return nil + }) + } + + _ = eg.Wait() +} + +var types = historyTypes{ { "acknowledgement", "icinga_acknowledgements", From 1141058b9cf73b8d3f507a0b83ac1e534bb35b82 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 20 Aug 2021 19:15:03 +0200 Subject: [PATCH 009/103] cmd/ido2icingadb: split main() --- cmd/ido2icingadb/main.go | 372 ++++++++++++++++++++------------------- 1 file changed, 192 insertions(+), 180 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 381a85f7..72f48c29 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -46,22 +46,9 @@ func run() int { return 2 } - cf, err := os.Open(f.Config) - if err != nil { - fmt.Fprintf(os.Stderr, "can't open config file: %s\n", err.Error()) - return 2 - } - - c := &Config{} - - { - err := yaml.NewDecoder(cf).Decode(c) - cf.Close() - - if err != nil { - fmt.Fprintf(os.Stderr, "can't parse config file: %s\n", err.Error()) - return 2 - } + c, ex := parseConfig(f) + if c == nil { + return ex } logger, _ := zap.NewDevelopmentConfig().Build() @@ -71,178 +58,53 @@ func run() int { log.Info("Starting IDO to Icinga DB history migration") - log.Info("Connecting to databases") - var ido, idb *icingadb.DB - - { - eg, _ := errgroup.WithContext(context.Background()) - - eg.Go(func() error { - ido = connect(log, "IDO", &c.IDO) - return nil - }) - - eg.Go(func() error { - idb = connect(log, "Icinga DB", &c.IcingaDB) - return nil - }) - - _ = eg.Wait() - } - - types.forEach(func(ht *historyType) { - tx, err := ido.BeginTxx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) - if err != nil { - log.Fatalf("%+v", errors.Wrap(err, "can't begin snapshot transaction")) - } - - ht.snapshot = tx - }) + ido, idb := connectAll(log, c) + startIdoTx(log, ido) log.Info("Computing progress") - types.forEach(func(ht *historyType) { - err := ht.snapshot.Get( - &ht.total, - "SELECT COUNT(*) FROM "+ht.idoTable+" xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id", - ) - if err != nil { - log.Fatalf("%+v", errors.Wrap(err, "can't count query")) - } - - log.With("type", ht.name, "amount", ht.total).Info("Counted total IDO events") - }) - + countIdoHistory(log) log.Sync() - - { - progress := mpb.New() - for i := range types { - typ := &types[i] - - typ.bar = progress.AddBar( - typ.total, - mpb.BarFillerClearOnComplete(), - mpb.PrependDecorators( - decor.Name(typ.name, decor.WC{W: len(typ.name) + 1, C: decor.DidentRight}), - decor.Percentage(decor.WC{W: 5}), - ), - mpb.AppendDecorators(decor.EwmaETA(decor.ET_STYLE_GO, 6000000, decor.WC{W: 4})), - ) - } - - types.forEach(func(ht *historyType) { - if ht.total == 0 { - ht.bar.SetTotal(ht.bar.Current(), true) - return - } - - query := "SELECT xh." + - strings.Join(append(append([]string(nil), ht.idoColumns...), ht.idoIdColumn), ", xh.") + " id FROM " + - ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + - ht.idoIdColumn + " > ? ORDER BY xh." + ht.idoIdColumn + " LIMIT 10000" - - stmt, err := ht.snapshot.Preparex(query) - if err != nil { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - defer stmt.Close() - - var lastRowsLen int - var lastQuery string - var lastStmt *sqlx.Stmt - start := time.Now() - - defer func() { - if lastStmt != nil { - lastStmt.Close() - } - }() - - Queries: - for { - var rows []ProgressRow - if err := stmt.Select(&rows, ht.lastId); err != nil { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) - } - - if len(rows) < 1 { - break - } - - if len(rows) != lastRowsLen { - if lastStmt != nil { - lastStmt.Close() - } - - buf := &bytes.Buffer{} - fmt.Fprintf(buf, "SELECT %s FROM %s WHERE %s IN (?", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn) - - for i := 1; i < len(rows); i++ { - buf.Write([]byte(",?")) - } - - buf.Write([]byte(")")) - lastRowsLen = len(rows) - lastQuery = buf.String() - - var err error - lastStmt, err = idb.Preparex(lastQuery) - - if err != nil { - log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - } - - ids := make([]interface{}, 0, len(rows)) - converted := make([]convertedId, 0, len(rows)) - - for _, row := range rows { - conv := ht.convertId(row, c.Icinga2.Env) - ids = append(ids, conv) - converted = append(converted, convertedId{row.Id, conv}) - } - - var present [][]byte - if err := lastStmt.Select(&present, ids...); err != nil { - log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't perform query")) - } - - presentSet := map[[20]byte]struct{}{} - for _, row := range present { - var key [20]byte - copy(key[:], row) - presentSet[key] = struct{}{} - } - - for _, conv := range converted { - var key [20]byte - copy(key[:], conv.idb) - - if _, ok := presentSet[key]; !ok { - break Queries - } - - ht.lastId = conv.ido - } - - prev := start - now := time.Now() - start = now - - ht.bar.IncrBy(len(rows)) - ht.bar.DecoratorEwmaUpdate(now.Sub(prev)) - } - - ht.bar.SetTotal(ht.bar.Current(), true) - }) - - progress.Wait() - } + computeProgress(log, c, idb) return internal.ExitSuccess } +func parseConfig(f *Flags) (*Config, int) { + cf, err := os.Open(f.Config) + if err != nil { + fmt.Fprintf(os.Stderr, "can't open config file: %s\n", err.Error()) + return nil, 2 + } + defer cf.Close() + + c := &Config{} + if err := yaml.NewDecoder(cf).Decode(c); err != nil { + fmt.Fprintf(os.Stderr, "can't parse config file: %s\n", err.Error()) + return nil, 2 + } + + return c, -1 +} + +func connectAll(log *zap.SugaredLogger, c *Config) (ido, idb *icingadb.DB) { + log.Info("Connecting to databases") + eg, _ := errgroup.WithContext(context.Background()) + + eg.Go(func() error { + ido = connect(log, "IDO", &c.IDO) + return nil + }) + + eg.Go(func() error { + idb = connect(log, "Icinga DB", &c.IcingaDB) + return nil + }) + + _ = eg.Wait() + return +} + func connect(log *zap.SugaredLogger, which string, cfg *config.Database) *icingadb.DB { db, err := cfg.Open(log) if err != nil { @@ -255,3 +117,153 @@ func connect(log *zap.SugaredLogger, which string, cfg *config.Database) *icinga return db } + +func startIdoTx(log *zap.SugaredLogger, ido *icingadb.DB) { + types.forEach(func(ht *historyType) { + tx, err := ido.BeginTxx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't begin snapshot transaction")) + } + + ht.snapshot = tx + }) +} + +func countIdoHistory(log *zap.SugaredLogger) { + types.forEach(func(ht *historyType) { + err := ht.snapshot.Get( + &ht.total, + "SELECT COUNT(*) FROM "+ht.idoTable+" xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id", + ) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't count query")) + } + + log.With("type", ht.name, "amount", ht.total).Info("Counted total IDO events") + }) +} + +func computeProgress(log *zap.SugaredLogger, c *Config, idb *icingadb.DB) { + progress := mpb.New() + for i := range types { + typ := &types[i] + + typ.bar = progress.AddBar( + typ.total, + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators( + decor.Name(typ.name, decor.WC{W: len(typ.name) + 1, C: decor.DidentRight}), + decor.Percentage(decor.WC{W: 5}), + ), + mpb.AppendDecorators(decor.EwmaETA(decor.ET_STYLE_GO, 6000000, decor.WC{W: 4})), + ) + } + + types.forEach(func(ht *historyType) { + if ht.total == 0 { + ht.bar.SetTotal(ht.bar.Current(), true) + return + } + + query := "SELECT xh." + + strings.Join(append(append([]string(nil), ht.idoColumns...), ht.idoIdColumn), ", xh.") + " id FROM " + + ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + + ht.idoIdColumn + " > ? ORDER BY xh." + ht.idoIdColumn + " LIMIT 10000" + + stmt, err := ht.snapshot.Preparex(query) + if err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + defer stmt.Close() + + var lastRowsLen int + var lastQuery string + var lastStmt *sqlx.Stmt + start := time.Now() + + defer func() { + if lastStmt != nil { + lastStmt.Close() + } + }() + + Queries: + for { + var rows []ProgressRow + if err := stmt.Select(&rows, ht.lastId); err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + + if len(rows) < 1 { + break + } + + if len(rows) != lastRowsLen { + if lastStmt != nil { + lastStmt.Close() + } + + buf := &bytes.Buffer{} + fmt.Fprintf(buf, "SELECT %s FROM %s WHERE %s IN (?", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn) + + for i := 1; i < len(rows); i++ { + buf.Write([]byte(",?")) + } + + buf.Write([]byte(")")) + lastRowsLen = len(rows) + lastQuery = buf.String() + + var err error + lastStmt, err = idb.Preparex(lastQuery) + + if err != nil { + log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + } + + ids := make([]interface{}, 0, len(rows)) + converted := make([]convertedId, 0, len(rows)) + + for _, row := range rows { + conv := ht.convertId(row, c.Icinga2.Env) + ids = append(ids, conv) + converted = append(converted, convertedId{row.Id, conv}) + } + + var present [][]byte + if err := lastStmt.Select(&present, ids...); err != nil { + log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + + presentSet := map[[20]byte]struct{}{} + for _, row := range present { + var key [20]byte + copy(key[:], row) + presentSet[key] = struct{}{} + } + + for _, conv := range converted { + var key [20]byte + copy(key[:], conv.idb) + + if _, ok := presentSet[key]; !ok { + break Queries + } + + ht.lastId = conv.ido + } + + prev := start + now := time.Now() + start = now + + ht.bar.IncrBy(len(rows)) + ht.bar.DecoratorEwmaUpdate(now.Sub(prev)) + } + + ht.bar.SetTotal(ht.bar.Current(), true) + }) + + progress.Wait() +} From 1d111b7dd354ff72be683b2d81e8725c959685d5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 23 Aug 2021 16:05:28 +0200 Subject: [PATCH 010/103] cmd/ido2icingadb: prepare cache --- cmd/ido2icingadb/main.go | 34 +++++++++++++++++ cmd/ido2icingadb/misc.go | 79 +++++++++++++++++++++++++++++++++++----- go.mod | 1 + 3 files changed, 104 insertions(+), 10 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 72f48c29..dc5cd5ea 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -11,12 +11,14 @@ import ( "github.com/icinga/icingadb/pkg/icingadb" "github.com/jessevdk/go-flags" "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" "github.com/pkg/errors" "github.com/vbauerster/mpb/v6" "github.com/vbauerster/mpb/v6/decor" "go.uber.org/zap" "golang.org/x/sync/errgroup" "os" + "path" "strings" "time" ) @@ -25,6 +27,8 @@ import ( type Flags struct { // Config is the path to the config file. Config string `short:"c" long:"config" description:"path to config file" required:"true"` + // Cache is a (not necessarily yet existing) directory for caching. + Cache string `short:"t" long:"cache" description:"path for caching" required:"true"` } // Config defines the YAML config structure. @@ -58,6 +62,7 @@ func run() int { log.Info("Starting IDO to Icinga DB history migration") + mkCache(log, f) ido, idb := connectAll(log, c) startIdoTx(log, ido) @@ -87,6 +92,35 @@ func parseConfig(f *Flags) (*Config, int) { return c, -1 } +func mkCache(log *zap.SugaredLogger, f *Flags) { + log.Info("Preparing cache") + + if err := os.MkdirAll(f.Cache, 0700); err != nil { + log.With("dir", f.Cache).Fatalf("%+v", errors.Wrap(err, "can't create directory")) + } + + types.forEach(func(ht *historyType) { + if ht.cacheSchema == nil { + return + } + + file := path.Join(f.Cache, ht.name+".sqlite3") + var err error + + ht.cache, err = sqlx.Open("sqlite3", "file:"+file) + if err != nil { + log.With("file", file).Fatalf("%+v", errors.Wrap(err, "can't open SQLite database")) + } + + for _, ddl := range ht.cacheSchema { + if _, err := ht.cache.Exec(ddl); err != nil { + log.With("file", file, "ddl", ddl). + Fatalf("%+v", errors.Wrap(err, "can't import schema into SQLite database")) + } + } + }) +} + func connectAll(log *zap.SugaredLogger, c *Config) (ido, idb *icingadb.DB) { log.Info("Connecting to databases") eg, _ := errgroup.WithContext(context.Background()) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index f0859779..40717d35 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -83,10 +83,13 @@ type historyType struct { idbTable string idbIdColumn string convertId func(row ProgressRow, env string) []byte - snapshot *sqlx.Tx - total int64 - bar *mpb.Bar - lastId uint64 + cacheSchema []string + + cache *sqlx.DB + snapshot *sqlx.Tx + total int64 + bar *mpb.Bar + lastId uint64 } type historyTypes [6]historyType @@ -113,7 +116,19 @@ var types = historyTypes{ "history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('a', row.Id) }, - nil, 0, nil, 0, + []string{ + `CREATE TABLE IF NOT EXISTS ack_clear_set_time ( + acknowledgement_id INT PRIMARY KEY, + entry_time INT, + entry_time_usec INT +)`, + `CREATE TABLE IF NOT EXISTS last_ack_set_time ( + object_id INT PRIMARY KEY, + entry_time INT NOT NULL, + entry_time_usec INT NOT NULL +)`, + }, + nil, nil, 0, nil, 0, }, { "comment", @@ -123,7 +138,7 @@ var types = historyTypes{ "comment_history", "comment_id", func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, - nil, 0, nil, 0, + nil, nil, nil, 0, nil, 0, }, { "downtime", @@ -133,7 +148,7 @@ var types = historyTypes{ "downtime_history", "downtime_id", func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, - nil, 0, nil, 0, + nil, nil, nil, 0, nil, 0, }, { "flapping", @@ -143,7 +158,19 @@ var types = historyTypes{ "history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('f', row.Id) }, - nil, 0, nil, 0, + []string{ + `CREATE TABLE IF NOT EXISTS flapping_end_start_time ( + flappinghistory_id INT PRIMARY KEY, + event_time INT, + event_time_usec INT +)`, + `CREATE TABLE IF NOT EXISTS last_flapping_start_time ( + object_id INT PRIMARY KEY, + event_time INT NOT NULL, + event_time_usec INT NOT NULL +)`, + }, + nil, nil, 0, nil, 0, }, { "notification", @@ -153,7 +180,23 @@ var types = historyTypes{ "notification_history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('n', row.Id) }, - nil, 0, nil, 0, + []string{ + `CREATE TABLE IF NOT EXISTS previous_hard_state ( + notification_id INT PRIMARY KEY, + previous_hard_state INT NOT NULL +)`, + `CREATE TABLE IF NOT EXISTS next_hard_state ( + object_id INT PRIMARY KEY, + next_hard_state INT NOT NULL +)`, + `CREATE TABLE IF NOT EXISTS next_ids ( + object_id INT NOT NULL, + notification_id INT NOT NULL +)`, + "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", + "CREATE INDEX IF NOT EXISTS next_ids_notification_id ON next_ids(notification_id)", + }, + nil, nil, 0, nil, 0, }, { "state", @@ -163,6 +206,22 @@ var types = historyTypes{ "state_history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('s', row.Id) }, - nil, 0, nil, 0, + []string{ + `CREATE TABLE IF NOT EXISTS previous_hard_state ( + statehistory_id INT PRIMARY KEY, + previous_hard_state INT NOT NULL +)`, + `CREATE TABLE IF NOT EXISTS next_hard_state ( + object_id INT PRIMARY KEY, + next_hard_state INT NOT NULL +)`, + `CREATE TABLE IF NOT EXISTS next_ids ( + object_id INT NOT NULL, + statehistory_id INT NOT NULL +)`, + "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", + "CREATE INDEX IF NOT EXISTS next_ids_statehistory_id ON next_ids(statehistory_id)", + }, + nil, nil, 0, nil, 0, }, } diff --git a/go.mod b/go.mod index a550de01..7e45b8fc 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/jessevdk/go-flags v1.5.0 github.com/jmoiron/sqlx v1.3.5 github.com/lib/pq v1.10.6 + github.com/mattn/go-sqlite3 v1.14.6 github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd github.com/pkg/errors v0.9.1 github.com/ssgreg/journald v1.0.0 From b005b4c08b9b89da52e1f33463e5c1a22408c4d7 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 23 Aug 2021 18:11:34 +0200 Subject: [PATCH 011/103] cmd/ido2icingadb: make log a global var --- cmd/ido2icingadb/main.go | 30 +++++++++++++----------------- cmd/ido2icingadb/misc.go | 10 ++++++++++ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index dc5cd5ea..0211dc18 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -15,7 +15,6 @@ import ( "github.com/pkg/errors" "github.com/vbauerster/mpb/v6" "github.com/vbauerster/mpb/v6/decor" - "go.uber.org/zap" "golang.org/x/sync/errgroup" "os" "path" @@ -55,22 +54,19 @@ func run() int { return ex } - logger, _ := zap.NewDevelopmentConfig().Build() - - log := logger.Sugar() defer log.Sync() log.Info("Starting IDO to Icinga DB history migration") - mkCache(log, f) - ido, idb := connectAll(log, c) - startIdoTx(log, ido) + mkCache(f) + ido, idb := connectAll(c) + startIdoTx(ido) log.Info("Computing progress") - countIdoHistory(log) + countIdoHistory() log.Sync() - computeProgress(log, c, idb) + computeProgress(c, idb) return internal.ExitSuccess } @@ -92,7 +88,7 @@ func parseConfig(f *Flags) (*Config, int) { return c, -1 } -func mkCache(log *zap.SugaredLogger, f *Flags) { +func mkCache(f *Flags) { log.Info("Preparing cache") if err := os.MkdirAll(f.Cache, 0700); err != nil { @@ -121,17 +117,17 @@ func mkCache(log *zap.SugaredLogger, f *Flags) { }) } -func connectAll(log *zap.SugaredLogger, c *Config) (ido, idb *icingadb.DB) { +func connectAll(c *Config) (ido, idb *icingadb.DB) { log.Info("Connecting to databases") eg, _ := errgroup.WithContext(context.Background()) eg.Go(func() error { - ido = connect(log, "IDO", &c.IDO) + ido = connect("IDO", &c.IDO) return nil }) eg.Go(func() error { - idb = connect(log, "Icinga DB", &c.IcingaDB) + idb = connect("Icinga DB", &c.IcingaDB) return nil }) @@ -139,7 +135,7 @@ func connectAll(log *zap.SugaredLogger, c *Config) (ido, idb *icingadb.DB) { return } -func connect(log *zap.SugaredLogger, which string, cfg *config.Database) *icingadb.DB { +func connect(which string, cfg *config.Database) *icingadb.DB { db, err := cfg.Open(log) if err != nil { log.With("backend", which).Fatalf("%+v", errors.Wrap(err, "can't connect to database")) @@ -152,7 +148,7 @@ func connect(log *zap.SugaredLogger, which string, cfg *config.Database) *icinga return db } -func startIdoTx(log *zap.SugaredLogger, ido *icingadb.DB) { +func startIdoTx(ido *icingadb.DB) { types.forEach(func(ht *historyType) { tx, err := ido.BeginTxx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) if err != nil { @@ -163,7 +159,7 @@ func startIdoTx(log *zap.SugaredLogger, ido *icingadb.DB) { }) } -func countIdoHistory(log *zap.SugaredLogger) { +func countIdoHistory() { types.forEach(func(ht *historyType) { err := ht.snapshot.Get( &ht.total, @@ -177,7 +173,7 @@ func countIdoHistory(log *zap.SugaredLogger) { }) } -func computeProgress(log *zap.SugaredLogger, c *Config, idb *icingadb.DB) { +func computeProgress(c *Config, idb *icingadb.DB) { progress := mpb.New() for i := range types { typ := &types[i] diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 40717d35..02fe02b6 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -9,6 +9,7 @@ import ( "github.com/icinga/icingadb/pkg/icingadb/objectpacker" "github.com/jmoiron/sqlx" "github.com/vbauerster/mpb/v6" + "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -22,6 +23,15 @@ type convertedId struct { idb []byte } +var log = func() *zap.SugaredLogger { + logger, err := zap.NewDevelopmentConfig().Build() + if err != nil { + panic(err) + } + + return logger.Sugar() +}() + // mkDeterministicUuid returns a formally random UUID (v4) as follows: 11111122-3300-4455-4455-555555555555 // // 0: zeroed From d9381ec2fae52da0804779ce0886136755898901 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 24 Aug 2021 14:49:44 +0200 Subject: [PATCH 012/103] cmd/ido2icingadb: separate cache schema --- cmd/ido2icingadb/ack.go | 14 ++++++++ cmd/ido2icingadb/flapping.go | 14 ++++++++ cmd/ido2icingadb/misc.go | 60 +++----------------------------- cmd/ido2icingadb/notification.go | 18 ++++++++++ cmd/ido2icingadb/state.go | 18 ++++++++++ 5 files changed, 68 insertions(+), 56 deletions(-) create mode 100644 cmd/ido2icingadb/ack.go create mode 100644 cmd/ido2icingadb/flapping.go create mode 100644 cmd/ido2icingadb/notification.go create mode 100644 cmd/ido2icingadb/state.go diff --git a/cmd/ido2icingadb/ack.go b/cmd/ido2icingadb/ack.go new file mode 100644 index 00000000..da342f45 --- /dev/null +++ b/cmd/ido2icingadb/ack.go @@ -0,0 +1,14 @@ +package main + +var ackCacheSchema = []string{ + `CREATE TABLE IF NOT EXISTS ack_clear_set_time ( + acknowledgement_id INT PRIMARY KEY, + entry_time INT, + entry_time_usec INT +)`, + `CREATE TABLE IF NOT EXISTS last_ack_set_time ( + object_id INT PRIMARY KEY, + entry_time INT NOT NULL, + entry_time_usec INT NOT NULL +)`, +} diff --git a/cmd/ido2icingadb/flapping.go b/cmd/ido2icingadb/flapping.go new file mode 100644 index 00000000..2c7c5f3b --- /dev/null +++ b/cmd/ido2icingadb/flapping.go @@ -0,0 +1,14 @@ +package main + +var flappingCacheSchema = []string{ + `CREATE TABLE IF NOT EXISTS flapping_end_start_time ( + flappinghistory_id INT PRIMARY KEY, + event_time INT, + event_time_usec INT +)`, + `CREATE TABLE IF NOT EXISTS last_flapping_start_time ( + object_id INT PRIMARY KEY, + event_time INT NOT NULL, + event_time_usec INT NOT NULL +)`, +} diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 02fe02b6..3982d158 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -126,18 +126,7 @@ var types = historyTypes{ "history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('a', row.Id) }, - []string{ - `CREATE TABLE IF NOT EXISTS ack_clear_set_time ( - acknowledgement_id INT PRIMARY KEY, - entry_time INT, - entry_time_usec INT -)`, - `CREATE TABLE IF NOT EXISTS last_ack_set_time ( - object_id INT PRIMARY KEY, - entry_time INT NOT NULL, - entry_time_usec INT NOT NULL -)`, - }, + ackCacheSchema, nil, nil, 0, nil, 0, }, { @@ -168,18 +157,7 @@ var types = historyTypes{ "history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('f', row.Id) }, - []string{ - `CREATE TABLE IF NOT EXISTS flapping_end_start_time ( - flappinghistory_id INT PRIMARY KEY, - event_time INT, - event_time_usec INT -)`, - `CREATE TABLE IF NOT EXISTS last_flapping_start_time ( - object_id INT PRIMARY KEY, - event_time INT NOT NULL, - event_time_usec INT NOT NULL -)`, - }, + flappingCacheSchema, nil, nil, 0, nil, 0, }, { @@ -190,22 +168,7 @@ var types = historyTypes{ "notification_history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('n', row.Id) }, - []string{ - `CREATE TABLE IF NOT EXISTS previous_hard_state ( - notification_id INT PRIMARY KEY, - previous_hard_state INT NOT NULL -)`, - `CREATE TABLE IF NOT EXISTS next_hard_state ( - object_id INT PRIMARY KEY, - next_hard_state INT NOT NULL -)`, - `CREATE TABLE IF NOT EXISTS next_ids ( - object_id INT NOT NULL, - notification_id INT NOT NULL -)`, - "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", - "CREATE INDEX IF NOT EXISTS next_ids_notification_id ON next_ids(notification_id)", - }, + notificationCacheSchema, nil, nil, 0, nil, 0, }, { @@ -216,22 +179,7 @@ var types = historyTypes{ "state_history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('s', row.Id) }, - []string{ - `CREATE TABLE IF NOT EXISTS previous_hard_state ( - statehistory_id INT PRIMARY KEY, - previous_hard_state INT NOT NULL -)`, - `CREATE TABLE IF NOT EXISTS next_hard_state ( - object_id INT PRIMARY KEY, - next_hard_state INT NOT NULL -)`, - `CREATE TABLE IF NOT EXISTS next_ids ( - object_id INT NOT NULL, - statehistory_id INT NOT NULL -)`, - "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", - "CREATE INDEX IF NOT EXISTS next_ids_statehistory_id ON next_ids(statehistory_id)", - }, + stateCacheSchema, nil, nil, 0, nil, 0, }, } diff --git a/cmd/ido2icingadb/notification.go b/cmd/ido2icingadb/notification.go new file mode 100644 index 00000000..2632a71f --- /dev/null +++ b/cmd/ido2icingadb/notification.go @@ -0,0 +1,18 @@ +package main + +var notificationCacheSchema = []string{ + `CREATE TABLE IF NOT EXISTS previous_hard_state ( + notification_id INT PRIMARY KEY, + previous_hard_state INT NOT NULL +)`, + `CREATE TABLE IF NOT EXISTS next_hard_state ( + object_id INT PRIMARY KEY, + next_hard_state INT NOT NULL +)`, + `CREATE TABLE IF NOT EXISTS next_ids ( + object_id INT NOT NULL, + notification_id INT NOT NULL +)`, + "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", + "CREATE INDEX IF NOT EXISTS next_ids_notification_id ON next_ids(notification_id)", +} diff --git a/cmd/ido2icingadb/state.go b/cmd/ido2icingadb/state.go new file mode 100644 index 00000000..69210510 --- /dev/null +++ b/cmd/ido2icingadb/state.go @@ -0,0 +1,18 @@ +package main + +var stateCacheSchema = []string{ + `CREATE TABLE IF NOT EXISTS previous_hard_state ( + statehistory_id INT PRIMARY KEY, + previous_hard_state INT NOT NULL +)`, + `CREATE TABLE IF NOT EXISTS next_hard_state ( + object_id INT PRIMARY KEY, + next_hard_state INT NOT NULL +)`, + `CREATE TABLE IF NOT EXISTS next_ids ( + object_id INT NOT NULL, + statehistory_id INT NOT NULL +)`, + "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", + "CREATE INDEX IF NOT EXISTS next_ids_statehistory_id ON next_ids(statehistory_id)", +} From 4de76da2396ebc96d16c02ce56ff2df512bde72f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 24 Aug 2021 14:54:04 +0200 Subject: [PATCH 013/103] cmd/ido2icingadb: introduce historyType#setupBar() --- cmd/ido2icingadb/main.go | 13 +------------ cmd/ido2icingadb/misc.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 0211dc18..3b8290b9 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -14,7 +14,6 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/pkg/errors" "github.com/vbauerster/mpb/v6" - "github.com/vbauerster/mpb/v6/decor" "golang.org/x/sync/errgroup" "os" "path" @@ -176,17 +175,7 @@ func countIdoHistory() { func computeProgress(c *Config, idb *icingadb.DB) { progress := mpb.New() for i := range types { - typ := &types[i] - - typ.bar = progress.AddBar( - typ.total, - mpb.BarFillerClearOnComplete(), - mpb.PrependDecorators( - decor.Name(typ.name, decor.WC{W: len(typ.name) + 1, C: decor.DidentRight}), - decor.Percentage(decor.WC{W: 5}), - ), - mpb.AppendDecorators(decor.EwmaETA(decor.ET_STYLE_GO, 6000000, decor.WC{W: 4})), - ) + types[i].setupBar(progress) } types.forEach(func(ht *historyType) { diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 3982d158..dda131f5 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -9,6 +9,7 @@ import ( "github.com/icinga/icingadb/pkg/icingadb/objectpacker" "github.com/jmoiron/sqlx" "github.com/vbauerster/mpb/v6" + "github.com/vbauerster/mpb/v6/decor" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -102,6 +103,18 @@ type historyType struct { lastId uint64 } +func (ht *historyType) setupBar(progress *mpb.Progress) { + ht.bar = progress.AddBar( + ht.total, + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators( + decor.Name(ht.name, decor.WC{W: len(ht.name) + 1, C: decor.DidentRight}), + decor.Percentage(decor.WC{W: 5}), + ), + mpb.AppendDecorators(decor.EwmaETA(decor.ET_STYLE_GO, 6000000, decor.WC{W: 4})), + ) +} + type historyTypes [6]historyType func (ht *historyTypes) forEach(f func(*historyType)) { From d9d8fb7576eee11295450ca53317dda0f4eb6bd3 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 24 Aug 2021 14:57:02 +0200 Subject: [PATCH 014/103] cmd/ido2icingadb: make bulk a const --- cmd/ido2icingadb/main.go | 4 ++-- cmd/ido2icingadb/misc.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 3b8290b9..1ee05180 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -187,7 +187,7 @@ func computeProgress(c *Config, idb *icingadb.DB) { query := "SELECT xh." + strings.Join(append(append([]string(nil), ht.idoColumns...), ht.idoIdColumn), ", xh.") + " id FROM " + ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + - ht.idoIdColumn + " > ? ORDER BY xh." + ht.idoIdColumn + " LIMIT 10000" + ht.idoIdColumn + " > ? ORDER BY xh." + ht.idoIdColumn + " LIMIT ?" stmt, err := ht.snapshot.Preparex(query) if err != nil { @@ -209,7 +209,7 @@ func computeProgress(c *Config, idb *icingadb.DB) { Queries: for { var rows []ProgressRow - if err := stmt.Select(&rows, ht.lastId); err != nil { + if err := stmt.Select(&rows, ht.lastId, bulk); err != nil { log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) } diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index dda131f5..d02df8fb 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -24,6 +24,8 @@ type convertedId struct { idb []byte } +const bulk = 10000 + var log = func() *zap.SugaredLogger { logger, err := zap.NewDevelopmentConfig().Build() if err != nil { From 423c948d3084660ee9b0581285efe381767cc18f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 24 Aug 2021 15:02:51 +0200 Subject: [PATCH 015/103] cmd/ido2icingadb: introduce barIncrementer --- cmd/ido2icingadb/main.go | 9 ++------- cmd/ido2icingadb/misc.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 1ee05180..5fad27b3 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -198,7 +198,7 @@ func computeProgress(c *Config, idb *icingadb.DB) { var lastRowsLen int var lastQuery string var lastStmt *sqlx.Stmt - start := time.Now() + inc := barIncrementer{ht.bar, time.Now()} defer func() { if lastStmt != nil { @@ -273,12 +273,7 @@ func computeProgress(c *Config, idb *icingadb.DB) { ht.lastId = conv.ido } - prev := start - now := time.Now() - start = now - - ht.bar.IncrBy(len(rows)) - ht.bar.DecoratorEwmaUpdate(now.Sub(prev)) + inc.inc(len(rows)) } ht.bar.SetTotal(ht.bar.Current(), true) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index d02df8fb..054ada09 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -12,6 +12,7 @@ import ( "github.com/vbauerster/mpb/v6/decor" "go.uber.org/zap" "golang.org/x/sync/errgroup" + "time" ) type ProgressRow struct { @@ -24,6 +25,20 @@ type convertedId struct { idb []byte } +type barIncrementer struct { + bar *mpb.Bar + start time.Time +} + +func (bi *barIncrementer) inc(i int) { + prev := bi.start + now := time.Now() + bi.start = now + + bi.bar.IncrBy(i) + bi.bar.DecoratorEwmaUpdate(now.Sub(prev)) +} + const bulk = 10000 var log = func() *zap.SugaredLogger { From 63ab8d938e7488473b018600a11f4496584a182a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 24 Aug 2021 18:15:34 +0200 Subject: [PATCH 016/103] cmd/ido2icingadb: support SELECT xh.foo_bar->FooBar int --- cmd/ido2icingadb/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 5fad27b3..e06e1a65 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -11,6 +11,7 @@ import ( "github.com/icinga/icingadb/pkg/icingadb" "github.com/jessevdk/go-flags" "github.com/jmoiron/sqlx" + "github.com/jmoiron/sqlx/reflectx" _ "github.com/mattn/go-sqlite3" "github.com/pkg/errors" "github.com/vbauerster/mpb/v6" @@ -57,9 +58,9 @@ func run() int { log.Info("Starting IDO to Icinga DB history migration") - mkCache(f) ido, idb := connectAll(c) startIdoTx(ido) + mkCache(f, idb.Mapper) log.Info("Computing progress") @@ -87,7 +88,7 @@ func parseConfig(f *Flags) (*Config, int) { return c, -1 } -func mkCache(f *Flags) { +func mkCache(f *Flags, mapper *reflectx.Mapper) { log.Info("Preparing cache") if err := os.MkdirAll(f.Cache, 0700); err != nil { @@ -107,6 +108,8 @@ func mkCache(f *Flags) { log.With("file", file).Fatalf("%+v", errors.Wrap(err, "can't open SQLite database")) } + ht.cache.Mapper = mapper + for _, ddl := range ht.cacheSchema { if _, err := ht.cache.Exec(ddl); err != nil { log.With("file", file, "ddl", ddl). From 0704750f85a8619cab37c86ac259acec8391dbb5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 25 Aug 2021 18:37:05 +0200 Subject: [PATCH 017/103] cmd/ido2icingadb: generalize cache schemata --- cmd/ido2icingadb/ack.go | 14 ------------ cmd/ido2icingadb/cache.go | 38 ++++++++++++++++++++++++++++++++ cmd/ido2icingadb/flapping.go | 14 ------------ cmd/ido2icingadb/misc.go | 8 +++---- cmd/ido2icingadb/notification.go | 18 --------------- cmd/ido2icingadb/state.go | 18 --------------- 6 files changed, 42 insertions(+), 68 deletions(-) delete mode 100644 cmd/ido2icingadb/ack.go create mode 100644 cmd/ido2icingadb/cache.go delete mode 100644 cmd/ido2icingadb/flapping.go delete mode 100644 cmd/ido2icingadb/notification.go delete mode 100644 cmd/ido2icingadb/state.go diff --git a/cmd/ido2icingadb/ack.go b/cmd/ido2icingadb/ack.go deleted file mode 100644 index da342f45..00000000 --- a/cmd/ido2icingadb/ack.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -var ackCacheSchema = []string{ - `CREATE TABLE IF NOT EXISTS ack_clear_set_time ( - acknowledgement_id INT PRIMARY KEY, - entry_time INT, - entry_time_usec INT -)`, - `CREATE TABLE IF NOT EXISTS last_ack_set_time ( - object_id INT PRIMARY KEY, - entry_time INT NOT NULL, - entry_time_usec INT NOT NULL -)`, -} diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go new file mode 100644 index 00000000..496f51a9 --- /dev/null +++ b/cmd/ido2icingadb/cache.go @@ -0,0 +1,38 @@ +package main + +var eventTimeCacheSchema = []string{ + // Icinga DB's flapping_history#start_time per flapping_end row (IDO's icinga_flappinghistory#flappinghistory_id). + // Similar for acknowledgements. + `CREATE TABLE IF NOT EXISTS end_start_time ( + history_id INT PRIMARY KEY, + event_time INT, + event_time_usec INT +)`, + // Helper table, the last start_time per icinga_statehistory#object_id. + `CREATE TABLE IF NOT EXISTS last_start_time ( + object_id INT PRIMARY KEY, + event_time INT NOT NULL, + event_time_usec INT NOT NULL +)`, +} + +var previousHardStateCacheSchema = []string{ + // Icinga DB's state_history#previous_hard_state per IDO's icinga_statehistory#statehistory_id. + // Similar for notifications. + `CREATE TABLE IF NOT EXISTS previous_hard_state ( + history_id INT PRIMARY KEY, + previous_hard_state INT NOT NULL +)`, + // Helper table, the current last_hard_state per icinga_statehistory#object_id. + `CREATE TABLE IF NOT EXISTS next_hard_state ( + object_id INT PRIMARY KEY, + next_hard_state INT NOT NULL +)`, + // Helper table for stashing icinga_statehistory#statehistory_id until last_hard_state changes. + `CREATE TABLE IF NOT EXISTS next_ids ( + object_id INT NOT NULL, + history_id INT NOT NULL +)`, + "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", + "CREATE INDEX IF NOT EXISTS next_ids_history_id ON next_ids(history_id)", +} diff --git a/cmd/ido2icingadb/flapping.go b/cmd/ido2icingadb/flapping.go deleted file mode 100644 index 2c7c5f3b..00000000 --- a/cmd/ido2icingadb/flapping.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -var flappingCacheSchema = []string{ - `CREATE TABLE IF NOT EXISTS flapping_end_start_time ( - flappinghistory_id INT PRIMARY KEY, - event_time INT, - event_time_usec INT -)`, - `CREATE TABLE IF NOT EXISTS last_flapping_start_time ( - object_id INT PRIMARY KEY, - event_time INT NOT NULL, - event_time_usec INT NOT NULL -)`, -} diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 054ada09..6d2e50f1 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -156,7 +156,7 @@ var types = historyTypes{ "history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('a', row.Id) }, - ackCacheSchema, + eventTimeCacheSchema, nil, nil, 0, nil, 0, }, { @@ -187,7 +187,7 @@ var types = historyTypes{ "history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('f', row.Id) }, - flappingCacheSchema, + eventTimeCacheSchema, nil, nil, 0, nil, 0, }, { @@ -198,7 +198,7 @@ var types = historyTypes{ "notification_history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('n', row.Id) }, - notificationCacheSchema, + previousHardStateCacheSchema, nil, nil, 0, nil, 0, }, { @@ -209,7 +209,7 @@ var types = historyTypes{ "state_history", "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('s', row.Id) }, - stateCacheSchema, + previousHardStateCacheSchema, nil, nil, 0, nil, 0, }, } diff --git a/cmd/ido2icingadb/notification.go b/cmd/ido2icingadb/notification.go deleted file mode 100644 index 2632a71f..00000000 --- a/cmd/ido2icingadb/notification.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -var notificationCacheSchema = []string{ - `CREATE TABLE IF NOT EXISTS previous_hard_state ( - notification_id INT PRIMARY KEY, - previous_hard_state INT NOT NULL -)`, - `CREATE TABLE IF NOT EXISTS next_hard_state ( - object_id INT PRIMARY KEY, - next_hard_state INT NOT NULL -)`, - `CREATE TABLE IF NOT EXISTS next_ids ( - object_id INT NOT NULL, - notification_id INT NOT NULL -)`, - "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", - "CREATE INDEX IF NOT EXISTS next_ids_notification_id ON next_ids(notification_id)", -} diff --git a/cmd/ido2icingadb/state.go b/cmd/ido2icingadb/state.go deleted file mode 100644 index 69210510..00000000 --- a/cmd/ido2icingadb/state.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -var stateCacheSchema = []string{ - `CREATE TABLE IF NOT EXISTS previous_hard_state ( - statehistory_id INT PRIMARY KEY, - previous_hard_state INT NOT NULL -)`, - `CREATE TABLE IF NOT EXISTS next_hard_state ( - object_id INT PRIMARY KEY, - next_hard_state INT NOT NULL -)`, - `CREATE TABLE IF NOT EXISTS next_ids ( - object_id INT NOT NULL, - statehistory_id INT NOT NULL -)`, - "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", - "CREATE INDEX IF NOT EXISTS next_ids_statehistory_id ON next_ids(statehistory_id)", -} From 6ae22d5fec8828eb0bc5db8806f45c2b399a2437 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 25 Aug 2021 19:40:15 +0200 Subject: [PATCH 018/103] cmd/ido2icingadb: fill cache --- cmd/ido2icingadb/cache.go | 280 +++++++++++++++++++++++++++++++++++++- cmd/ido2icingadb/main.go | 20 +++ cmd/ido2icingadb/misc.go | 58 +++++++- 3 files changed, 354 insertions(+), 4 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 496f51a9..62d02f1c 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -1,8 +1,16 @@ package main +import ( + "database/sql" + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + "math" + "strings" + "time" +) + var eventTimeCacheSchema = []string{ // Icinga DB's flapping_history#start_time per flapping_end row (IDO's icinga_flappinghistory#flappinghistory_id). - // Similar for acknowledgements. `CREATE TABLE IF NOT EXISTS end_start_time ( history_id INT PRIMARY KEY, event_time INT, @@ -16,9 +24,89 @@ var eventTimeCacheSchema = []string{ )`, } +// buildEventTimeCache rationale: +// +// Icinga DB's flapping_history#id always needs start_time. flapping_end rows would need an IDO subquery for that. +// That would make the IDO reading even slower than the Icinga DB writing. +// Therefore: Stream IDO's icinga_flappinghistory once, compute flapping_history#start_time +// and cache it into an SQLite database. Then steam from that database and the IDO. +// +// Similar for acknowledgements. +func buildEventTimeCache(ht *historyType, idoColumns []string) { + chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func()) { + var checkpoint struct { + Cnt int64 + MaxId sql.NullInt64 + } + cacheGet(*tx, &checkpoint, "SELECT COUNT(*) cnt, MAX(history_id) max_id FROM end_start_time") + + ht.bar.SetCurrent(checkpoint.Cnt * 2) + var inc *barIncrementer + + streamIdoQuery( + ht.snapshot, + "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ + ht.idoIdColumn+" > ? ORDER BY xh."+ht.idoIdColumn, + []interface{}{checkpoint.MaxId.Int64}, + func() { inc = &barIncrementer{ht.bar, time.Now()} }, + func(idoRow *struct { + Id uint64 + EventTime int64 + EventTimeUsec uint32 + EventIsStart uint8 + ObjectId uint64 + }) { + if idoRow.EventIsStart == 0 { + var lst []struct { + EventTime int64 + EventTimeUsec uint32 + } + cacheSelect( + *tx, &lst, + "SELECT event_time, event_time_usec FROM last_start_time WHERE object_id=?", idoRow.ObjectId, + ) + + if len(lst) > 0 { + cacheExec( + *tx, false, + "INSERT INTO end_start_time(history_id, event_time, event_time_usec) VALUES (?, ?, ?)", + idoRow.Id, lst[0].EventTime, lst[0].EventTimeUsec, + ) + + onDeleted(cacheExec( + *tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId, + )) + } else { + cacheExec( + *tx, false, + "INSERT INTO end_start_time(history_id, event_time, event_time_usec) "+ + "VALUES (?, NULL, NULL)", idoRow.Id, + ) + } + } else { + onDeleted(cacheExec(*tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId)) + + cacheExec( + *tx, false, + "INSERT INTO last_start_time(object_id, event_time, event_time_usec) VALUES (?, ?, ?)", + idoRow.ObjectId, idoRow.EventTime, idoRow.EventTimeUsec, + ) + } + + onNewUncommittedDml() + inc.inc(1) + }, + ) + + onDeleted(cacheExec(*tx, false, "DELETE FROM last_start_time")) + }) + + ht.bar.SetTotal(ht.bar.Current(), true) +} + var previousHardStateCacheSchema = []string{ // Icinga DB's state_history#previous_hard_state per IDO's icinga_statehistory#statehistory_id. - // Similar for notifications. `CREATE TABLE IF NOT EXISTS previous_hard_state ( history_id INT PRIMARY KEY, previous_hard_state INT NOT NULL @@ -36,3 +124,191 @@ var previousHardStateCacheSchema = []string{ "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", "CREATE INDEX IF NOT EXISTS next_ids_history_id ON next_ids(history_id)", } + +// buildPreviousHardStateCache rationale: +// +// Icinga DB's state_history#previous_hard_state would need a subquery. +// That make the IDO reading even slower than the Icinga DB writing. +// Therefore: Stream IDO's icinga_statehistory once, compute state_history#previous_hard_state +// and cache it into an SQLite database. Then steam from that database and the IDO. +// +// Similar for notifications. +func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { + chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func()) { + var nextIds struct { + Cnt int64 + MinId sql.NullInt64 + } + cacheGet(*tx, &nextIds, "SELECT COUNT(*) cnt, MIN(history_id) min_id FROM next_ids") + + var previousHardState struct{ Cnt int64 } + cacheGet(*tx, &previousHardState, "SELECT COUNT(*) cnt FROM previous_hard_state") + + var checkpoint int64 + if nextIds.MinId.Valid { + checkpoint = nextIds.MinId.Int64 + } else { + // next_ids contains the most recently processed IDs and is only empty if... + if previousHardState.Cnt == 0 { + // ... we didn't actually start yet... + checkpoint = math.MaxInt64 + } else { + // ... or we've already finished. + checkpoint = 0 + } + } + + ht.bar.SetCurrent(previousHardState.Cnt + nextIds.Cnt) + var inc *barIncrementer + + // We continue where we finished before. As we build the cache in reverse chronological order: + // 1. If the history grows between two migration trials, we won't migrate the difference. Workarounds: + // a. Start migration after Icinga DB is up and running. + // b. Remove the cache before the next migration trial. + // 2. If the history gets cleaned up between two migration trials, + // the difference either just doesn't appear in the cache or - if already there - will be ignored later. + + streamIdoQuery( + ht.snapshot, + "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ + ht.idoIdColumn+" < ? ORDER BY xh."+ht.idoIdColumn+" DESC", + []interface{}{checkpoint}, + func() { inc = &barIncrementer{ht.bar, time.Now()} }, + func(idoRow *struct { + Id uint64 + ObjectId uint64 + LastHardState uint8 + }) { + var nhs []struct{ NextHardState uint8 } + cacheSelect(*tx, &nhs, "SELECT next_hard_state FROM next_hard_state WHERE object_id=?", idoRow.ObjectId) + + if len(nhs) < 1 { + cacheExec( + *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", + idoRow.ObjectId, idoRow.LastHardState, + ) + + cacheExec( + *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", + idoRow.Id, idoRow.ObjectId, + ) + } else if idoRow.LastHardState == nhs[0].NextHardState { + cacheExec( + *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", + idoRow.Id, idoRow.ObjectId, + ) + } else { + cacheExec( + *tx, false, "INSERT INTO previous_hard_state(history_id, previous_hard_state) "+ + "SELECT history_id, ? FROM next_ids WHERE object_id=?", + idoRow.LastHardState, idoRow.ObjectId, + ) + + onDeleted(cacheExec(*tx, false, "DELETE FROM next_hard_state WHERE object_id=?", idoRow.ObjectId)) + onDeleted(cacheExec(*tx, false, "DELETE FROM next_ids WHERE object_id=?", idoRow.ObjectId)) + + cacheExec( + *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", + idoRow.ObjectId, idoRow.LastHardState, + ) + + cacheExec( + *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", + idoRow.Id, idoRow.ObjectId, + ) + } + + onNewUncommittedDml() + inc.inc(1) + }, + ) + + cacheExec( + *tx, false, "INSERT INTO previous_hard_state(history_id, previous_hard_state) "+ + "SELECT history_id, 99 FROM next_ids", + ) + + onDeleted(cacheExec(*tx, false, "DELETE FROM next_hard_state")) + onDeleted(cacheExec(*tx, false, "DELETE FROM next_ids")) + }) + + ht.bar.SetTotal(ht.bar.Current(), true) +} + +func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func())) { + logger := log.With("backend", "cache") + var deleted int64 + var inTx int + + tx, err := cache.Beginx() + if err != nil { + logger.Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) + } + + do( + &tx, + func(result sql.Result) { + if affected, err := result.RowsAffected(); err == nil { + deleted += affected + } else { + log.Errorf("%+v", errors.Wrap(err, "can't get affected rows")) + } + }, + func() { + inTx++ + if inTx == bulk { + if err := tx.Commit(); err != nil { + logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) + } + + var err error + + tx, err = cache.Beginx() + if err != nil { + logger.Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) + } + + inTx = 0 + } + }, + ) + + if err := tx.Commit(); err != nil { + logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) + } + + if deleted > 0 { + cacheExec(cache, true, "VACUUM") + } +} + +func cacheGet(cacheTx *sqlx.Tx, dest interface{}, query string, args ...interface{}) { + if err := cacheTx.Get(dest, query, args...); err != nil { + log.With("backend", "cache", "query", query, "args", args). + Errorf("%+v", errors.Wrap(err, "can't perform query")) + } +} + +func cacheSelect(cacheTx *sqlx.Tx, dest interface{}, query string, args ...interface{}) { + if err := cacheTx.Select(dest, query, args...); err != nil { + log.With("backend", "cache", "query", query, "args", args). + Errorf("%+v", errors.Wrap(err, "can't perform query")) + } +} + +func cacheExec(cache sqlx.Execer, allowFailure bool, dml string, args ...interface{}) sql.Result { + res, err := cache.Exec(dml, args...) + if err != nil { + logger := log.With("backend", "cache", "dml", dml, "args", args) + + level := logger.Fatalf + if allowFailure { + level = logger.Errorf + } + + level("%+v", errors.Wrap(err, "can't perform DML")) + } + + return res +} diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index e06e1a65..dc1d8e3b 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -68,6 +68,9 @@ func run() int { log.Sync() computeProgress(c, idb) + log.Info("Filling cache") + fillCache() + return internal.ExitSuccess } @@ -284,3 +287,20 @@ func computeProgress(c *Config, idb *icingadb.DB) { progress.Wait() } + +func fillCache() { + progress := mpb.New() + for i := range types { + if types[i].cacheFiller != nil { + types[i].setupBar(progress) + } + } + + types.forEach(func(ht *historyType) { + if ht.cacheFiller != nil { + ht.cacheFiller(ht) + } + }) + + progress.Wait() +} diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 6d2e50f1..1858b9b3 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -8,10 +8,12 @@ import ( "github.com/google/uuid" "github.com/icinga/icingadb/pkg/icingadb/objectpacker" "github.com/jmoiron/sqlx" + "github.com/pkg/errors" "github.com/vbauerster/mpb/v6" "github.com/vbauerster/mpb/v6/decor" "go.uber.org/zap" "golang.org/x/sync/errgroup" + "reflect" "time" ) @@ -103,6 +105,37 @@ func calcObjectId(env, name1 string) []byte { return hashAny([2]string{env, name1}) } +func streamIdoQuery(snapshot *sqlx.Tx, query string, args []interface{}, onResultSetReady func(), onRow interface{}) { + vOnRow := reflect.ValueOf(onRow) // TODO: make onRow generic[T] one nice day + + tRow := vOnRow.Type(). // func(idoRow *T) + In(0). // *T + Elem() // T + + rows, err := snapshot.Queryx(query, args...) + if err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + defer rows.Close() + + onResultSetReady() + + for { + if rows.Next() { + vRow := reflect.New(tRow) + if err := rows.StructScan(vRow.Interface()); err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't scan result set")) + } + + vOnRow.Call([]reflect.Value{vRow}) + } else if err := rows.Err(); err == nil { + break + } else { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't fetch result set")) + } + } +} + type historyType struct { name string idoTable string @@ -112,6 +145,7 @@ type historyType struct { idbIdColumn string convertId func(row ProgressRow, env string) []byte cacheSchema []string + cacheFiller func(*historyType) cache *sqlx.DB snapshot *sqlx.Tx @@ -157,6 +191,12 @@ var types = historyTypes{ "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('a', row.Id) }, eventTimeCacheSchema, + func(ht *historyType) { + buildEventTimeCache(ht, []string{ + "xh.acknowledgement_id id", "UNIX_TIMESTAMP(xh.entry_time) event_time", + "xh.entry_time_usec event_time_usec", "xh.acknowledgement_type event_is_start", "xh.object_id", + }) + }, nil, nil, 0, nil, 0, }, { @@ -167,7 +207,7 @@ var types = historyTypes{ "comment_history", "comment_id", func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, - nil, nil, nil, 0, nil, 0, + nil, nil, nil, nil, 0, nil, 0, }, { "downtime", @@ -177,7 +217,7 @@ var types = historyTypes{ "downtime_history", "downtime_id", func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, - nil, nil, nil, 0, nil, 0, + nil, nil, nil, nil, 0, nil, 0, }, { "flapping", @@ -188,6 +228,12 @@ var types = historyTypes{ "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('f', row.Id) }, eventTimeCacheSchema, + func(ht *historyType) { + buildEventTimeCache(ht, []string{ + "xh.flappinghistory_id id", "UNIX_TIMESTAMP(xh.event_time) event_time", + "xh.event_time_usec", "xh.event_type-1000 event_is_start", "xh.object_id", + }) + }, nil, nil, 0, nil, 0, }, { @@ -199,6 +245,11 @@ var types = historyTypes{ "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('n', row.Id) }, previousHardStateCacheSchema, + func(ht *historyType) { + buildPreviousHardStateCache(ht, []string{ + "xh.notification_id id", "xh.object_id", "xh.state last_hard_state", + }) + }, nil, nil, 0, nil, 0, }, { @@ -210,6 +261,9 @@ var types = historyTypes{ "id", func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('s', row.Id) }, previousHardStateCacheSchema, + func(ht *historyType) { + buildPreviousHardStateCache(ht, []string{"xh.statehistory_id id", "xh.object_id", "xh.last_hard_state"}) + }, nil, nil, 0, nil, 0, }, } From 6b583242ebe3c7485a8623396e4b960f8fac3e92 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 6 Sep 2021 13:49:22 +0200 Subject: [PATCH 019/103] cmd/ido2icingadb: chunk all large queries --- cmd/ido2icingadb/cache.go | 160 ++++++++++++++++++++------------------ cmd/ido2icingadb/main.go | 23 +----- cmd/ido2icingadb/misc.go | 46 ++++++----- 3 files changed, 114 insertions(+), 115 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 62d02f1c..3ecc1ea9 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -41,61 +41,67 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { cacheGet(*tx, &checkpoint, "SELECT COUNT(*) cnt, MAX(history_id) max_id FROM end_start_time") ht.bar.SetCurrent(checkpoint.Cnt * 2) - var inc *barIncrementer + inc := barIncrementer{ht.bar, time.Now()} - streamIdoQuery( + sliceIdoHistory( ht.snapshot, "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ - ht.idoIdColumn+" > ? ORDER BY xh."+ht.idoIdColumn, - []interface{}{checkpoint.MaxId.Int64}, - func() { inc = &barIncrementer{ht.bar, time.Now()} }, - func(idoRow *struct { + ht.idoIdColumn+" > ? ORDER BY xh."+ht.idoIdColumn+" LIMIT ?", + checkpoint.MaxId.Int64, + func(idoRows []struct { Id uint64 EventTime int64 EventTimeUsec uint32 EventIsStart uint8 ObjectId uint64 - }) { - if idoRow.EventIsStart == 0 { - var lst []struct { - EventTime int64 - EventTimeUsec uint32 - } - cacheSelect( - *tx, &lst, - "SELECT event_time, event_time_usec FROM last_start_time WHERE object_id=?", idoRow.ObjectId, - ) - - if len(lst) > 0 { - cacheExec( - *tx, false, - "INSERT INTO end_start_time(history_id, event_time, event_time_usec) VALUES (?, ?, ?)", - idoRow.Id, lst[0].EventTime, lst[0].EventTimeUsec, + }) (checkpoint interface{}) { + for _, idoRow := range idoRows { + if idoRow.EventIsStart == 0 { + var lst []struct { + EventTime int64 + EventTimeUsec uint32 + } + cacheSelect( + *tx, &lst, "SELECT event_time, event_time_usec FROM last_start_time WHERE object_id=?", + idoRow.ObjectId, ) + if len(lst) > 0 { + cacheExec( + *tx, false, + "INSERT INTO end_start_time(history_id, event_time, event_time_usec) VALUES (?, ?, ?)", + idoRow.Id, lst[0].EventTime, lst[0].EventTimeUsec, + ) + + onDeleted(cacheExec( + *tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId, + )) + } else { + cacheExec( + *tx, false, + "INSERT INTO end_start_time(history_id, event_time, event_time_usec) "+ + "VALUES (?, NULL, NULL)", idoRow.Id, + ) + } + } else { onDeleted(cacheExec( *tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId, )) - } else { + cacheExec( *tx, false, - "INSERT INTO end_start_time(history_id, event_time, event_time_usec) "+ - "VALUES (?, NULL, NULL)", idoRow.Id, + "INSERT INTO last_start_time(object_id, event_time, event_time_usec) VALUES (?, ?, ?)", + idoRow.ObjectId, idoRow.EventTime, idoRow.EventTimeUsec, ) } - } else { - onDeleted(cacheExec(*tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId)) - cacheExec( - *tx, false, - "INSERT INTO last_start_time(object_id, event_time, event_time_usec) VALUES (?, ?, ?)", - idoRow.ObjectId, idoRow.EventTime, idoRow.EventTimeUsec, - ) + onNewUncommittedDml() + checkpoint = idoRow.Id } - onNewUncommittedDml() - inc.inc(1) + inc.inc(len(idoRows)) + return }, ) @@ -159,7 +165,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { } ht.bar.SetCurrent(previousHardState.Cnt + nextIds.Cnt) - var inc *barIncrementer + inc := barIncrementer{ht.bar, time.Now()} // We continue where we finished before. As we build the cache in reverse chronological order: // 1. If the history grows between two migration trials, we won't migrate the difference. Workarounds: @@ -168,59 +174,63 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { // 2. If the history gets cleaned up between two migration trials, // the difference either just doesn't appear in the cache or - if already there - will be ignored later. - streamIdoQuery( + sliceIdoHistory( ht.snapshot, "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ - ht.idoIdColumn+" < ? ORDER BY xh."+ht.idoIdColumn+" DESC", - []interface{}{checkpoint}, - func() { inc = &barIncrementer{ht.bar, time.Now()} }, - func(idoRow *struct { + ht.idoIdColumn+" < ? ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT ?", + checkpoint, + func(idoRows []struct { Id uint64 ObjectId uint64 LastHardState uint8 - }) { - var nhs []struct{ NextHardState uint8 } - cacheSelect(*tx, &nhs, "SELECT next_hard_state FROM next_hard_state WHERE object_id=?", idoRow.ObjectId) + }) (checkpoint interface{}) { + for _, idoRow := range idoRows { + var nhs []struct{ NextHardState uint8 } + cacheSelect(*tx, &nhs, "SELECT next_hard_state FROM next_hard_state WHERE object_id=?", idoRow.ObjectId) - if len(nhs) < 1 { - cacheExec( - *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", - idoRow.ObjectId, idoRow.LastHardState, - ) + if len(nhs) < 1 { + cacheExec( + *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", + idoRow.ObjectId, idoRow.LastHardState, + ) - cacheExec( - *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", - idoRow.Id, idoRow.ObjectId, - ) - } else if idoRow.LastHardState == nhs[0].NextHardState { - cacheExec( - *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", - idoRow.Id, idoRow.ObjectId, - ) - } else { - cacheExec( - *tx, false, "INSERT INTO previous_hard_state(history_id, previous_hard_state) "+ - "SELECT history_id, ? FROM next_ids WHERE object_id=?", - idoRow.LastHardState, idoRow.ObjectId, - ) + cacheExec( + *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", + idoRow.Id, idoRow.ObjectId, + ) + } else if idoRow.LastHardState == nhs[0].NextHardState { + cacheExec( + *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", + idoRow.Id, idoRow.ObjectId, + ) + } else { + cacheExec( + *tx, false, "INSERT INTO previous_hard_state(history_id, previous_hard_state) "+ + "SELECT history_id, ? FROM next_ids WHERE object_id=?", + idoRow.LastHardState, idoRow.ObjectId, + ) - onDeleted(cacheExec(*tx, false, "DELETE FROM next_hard_state WHERE object_id=?", idoRow.ObjectId)) - onDeleted(cacheExec(*tx, false, "DELETE FROM next_ids WHERE object_id=?", idoRow.ObjectId)) + onDeleted(cacheExec(*tx, false, "DELETE FROM next_hard_state WHERE object_id=?", idoRow.ObjectId)) + onDeleted(cacheExec(*tx, false, "DELETE FROM next_ids WHERE object_id=?", idoRow.ObjectId)) - cacheExec( - *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", - idoRow.ObjectId, idoRow.LastHardState, - ) + cacheExec( + *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", + idoRow.ObjectId, idoRow.LastHardState, + ) - cacheExec( - *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", - idoRow.Id, idoRow.ObjectId, - ) + cacheExec( + *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", + idoRow.Id, idoRow.ObjectId, + ) + } + + onNewUncommittedDml() + checkpoint = idoRow.Id } - onNewUncommittedDml() - inc.inc(1) + inc.inc(len(idoRows)) + return }, ) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index dc1d8e3b..e7a03604 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -195,12 +195,6 @@ func computeProgress(c *Config, idb *icingadb.DB) { ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + ht.idoIdColumn + " > ? ORDER BY xh." + ht.idoIdColumn + " LIMIT ?" - stmt, err := ht.snapshot.Preparex(query) - if err != nil { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - defer stmt.Close() - var lastRowsLen int var lastQuery string var lastStmt *sqlx.Stmt @@ -212,17 +206,7 @@ func computeProgress(c *Config, idb *icingadb.DB) { } }() - Queries: - for { - var rows []ProgressRow - if err := stmt.Select(&rows, ht.lastId, bulk); err != nil { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) - } - - if len(rows) < 1 { - break - } - + sliceIdoHistory(ht.snapshot, query, 0, func(rows []ProgressRow) (checkpoint interface{}) { if len(rows) != lastRowsLen { if lastStmt != nil { lastStmt.Close() @@ -273,14 +257,15 @@ func computeProgress(c *Config, idb *icingadb.DB) { copy(key[:], conv.idb) if _, ok := presentSet[key]; !ok { - break Queries + return nil } ht.lastId = conv.ido } inc.inc(len(rows)) - } + return ht.lastId + }) ht.bar.SetTotal(ht.bar.Current(), true) }) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 1858b9b3..d6600564 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -105,34 +105,38 @@ func calcObjectId(env, name1 string) []byte { return hashAny([2]string{env, name1}) } -func streamIdoQuery(snapshot *sqlx.Tx, query string, args []interface{}, onResultSetReady func(), onRow interface{}) { - vOnRow := reflect.ValueOf(onRow) // TODO: make onRow generic[T] one nice day +func sliceIdoHistory(snapshot *sqlx.Tx, query string, checkpoint, onRows interface{}) { + vOnRows := reflect.ValueOf(onRows) // TODO: make onRows generic[T] one nice day - tRow := vOnRow.Type(). // func(idoRow *T) - In(0). // *T - Elem() // T + tRows := vOnRows.Type(). // func(rows []T) (checkpoint interface{}) + In(0) // []T - rows, err := snapshot.Queryx(query, args...) + vNewRows := reflect.New(tRows) + rowsPtr := vNewRows.Interface() + vRows := vNewRows.Elem() + onRowsArgs := [1]reflect.Value{vRows} + vZeroRows := reflect.Zero(tRows) + + stmt, err := snapshot.Preparex(query) if err != nil { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) } - defer rows.Close() - - onResultSetReady() + defer stmt.Close() for { - if rows.Next() { - vRow := reflect.New(tRow) - if err := rows.StructScan(vRow.Interface()); err != nil { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't scan result set")) - } - - vOnRow.Call([]reflect.Value{vRow}) - } else if err := rows.Err(); err == nil { - break - } else { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't fetch result set")) + if err := stmt.Select(rowsPtr, checkpoint, bulk); err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) } + + if vRows.Len() < 1 { + break + } + + if checkpoint = vOnRows.Call(onRowsArgs[:])[0].Interface(); checkpoint == nil { + break + } + + vRows.Set(vZeroRows) } } From 5396afe32afd5e6aff4fdb73286310665a3bcd29 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Oct 2021 15:19:13 +0200 Subject: [PATCH 020/103] cmd/ido2icingadb: actually migrate --- cmd/ido2icingadb/cache.go | 20 +- cmd/ido2icingadb/convert.go | 861 ++++++++++++++++++++++++++++++++++ cmd/ido2icingadb/main.go | 162 ++++++- cmd/ido2icingadb/misc.go | 131 +++++- cmd/ido2icingadb/misc_test.go | 6 +- 5 files changed, 1137 insertions(+), 43 deletions(-) create mode 100644 cmd/ido2icingadb/convert.go diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 3ecc1ea9..82cc4acf 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -13,8 +13,8 @@ var eventTimeCacheSchema = []string{ // Icinga DB's flapping_history#start_time per flapping_end row (IDO's icinga_flappinghistory#flappinghistory_id). `CREATE TABLE IF NOT EXISTS end_start_time ( history_id INT PRIMARY KEY, - event_time INT, - event_time_usec INT + event_time INT NOT NULL, + event_time_usec INT NOT NULL )`, // Helper table, the last start_time per icinga_statehistory#object_id. `CREATE TABLE IF NOT EXISTS last_start_time ( @@ -48,7 +48,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" > ? ORDER BY xh."+ht.idoIdColumn+" LIMIT ?", - checkpoint.MaxId.Int64, + nil, checkpoint.MaxId.Int64, func(idoRows []struct { Id uint64 EventTime int64 @@ -77,12 +77,6 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { onDeleted(cacheExec( *tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId, )) - } else { - cacheExec( - *tx, false, - "INSERT INTO end_start_time(history_id, event_time, event_time_usec) "+ - "VALUES (?, NULL, NULL)", idoRow.Id, - ) } } else { onDeleted(cacheExec( @@ -179,7 +173,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" < ? ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT ?", - checkpoint, + nil, checkpoint, func(idoRows []struct { Id uint64 ObjectId uint64 @@ -293,8 +287,10 @@ func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onDeleted func(sql.Resul } } -func cacheGet(cacheTx *sqlx.Tx, dest interface{}, query string, args ...interface{}) { - if err := cacheTx.Get(dest, query, args...); err != nil { +func cacheGet(cache interface { + Get(dest interface{}, query string, args ...interface{}) error +}, dest interface{}, query string, args ...interface{}) { + if err := cache.Get(dest, query, args...); err != nil { log.With("backend", "cache", "query", query, "args", args). Errorf("%+v", errors.Wrap(err, "can't perform query")) } diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go new file mode 100644 index 00000000..e7642060 --- /dev/null +++ b/cmd/ido2icingadb/convert.go @@ -0,0 +1,861 @@ +package main + +import ( + "database/sql" + "github.com/icinga/icingadb/pkg/contracts" + v1 "github.com/icinga/icingadb/pkg/icingadb/v1" + "github.com/icinga/icingadb/pkg/icingadb/v1/history" + icingadbTypes "github.com/icinga/icingadb/pkg/types" + "github.com/icinga/icingadb/pkg/utils" + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + "strings" + "time" +) + +const acknowledgementMigrationQuery = "SELECT ah.acknowledgement_id, UNIX_TIMESTAMP(ah.entry_time) entry_time, " + + "ah.entry_time_usec, ah.acknowledgement_type, ah.author_name, ah.comment_data, ah.is_sticky, " + + "ah.persistent_comment, UNIX_TIMESTAMP(ah.end_time) end_time, o.objecttype_id, o.name1, " + + "IFNULL(o.name2, '') name2 " + + "FROM icinga_acknowledgements ah USE INDEX (PRIMARY) " + + "INNER JOIN icinga_objects o ON o.object_id=ah.object_id " + + "WHERE ah.acknowledgement_id > ? " + // where we were interrupted + "ORDER BY ah.acknowledgement_id " + // allows computeProgress() not to check all IDO rows for whether migrated + "LIMIT ?" + +// AckClear updates an already migrated ack event with the clear event info. +type AckClear struct { + Id icingadbTypes.Binary + ClearTime icingadbTypes.UnixMilli +} + +// Assert interface compliance. +var _ contracts.TableNamer = (*AckClear)(nil) + +// TableName implements the contracts.TableNamer interface. +func (*AckClear) TableName() string { + return "acknowledgement_history" +} + +type acknowledgementRow = struct { + AcknowledgementId uint64 + EntryTime int64 + EntryTimeUsec uint32 + AcknowledgementType uint8 + AuthorName sql.NullString + CommentData sql.NullString + IsSticky uint8 + PersistentComment uint8 + EndTime sql.NullInt64 + ObjecttypeId uint8 + Name1 string + Name2 string +} + +func convertAcknowledgementRows( + env string, envId, endpointId icingadbTypes.Binary, + selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []acknowledgementRow, +) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) { + if len(idoRows) < 1 { + return + } + + var cached []struct { + HistoryId uint64 + EventTime int64 + EventTimeUsec uint32 + } + selectCache( + &cached, "SELECT history_id, event_time, event_time_usec FROM end_start_time WHERE history_id BETWEEN ? AND ?", + idoRows[0].AcknowledgementId, idoRows[len(idoRows)-1].AcknowledgementId, + ) + + // Needed for set time (see below). + cachedById := make(map[uint64]icingadbTypes.UnixMilli, len(cached)) + for _, c := range cached { + cachedById[c.HistoryId] = convertTime(c.EventTime, c.EventTimeUsec) + } + + var acknowledgementHistory, acknowledgementHistoryUpdates, allHistory []interface{} + for _, row := range idoRows { + ts := convertTime(row.EntryTime, row.EntryTimeUsec) + + // Needed for ID (see below). + var set icingadbTypes.UnixMilli + if row.AcknowledgementType == 0 { // clear + var ok bool + set, ok = cachedById[row.AcknowledgementId] + + if !ok { + continue + } + } else { + set = ts + } + + name := row.Name1 + if row.Name2 != "" { + name += "!" + row.Name2 + } + + id := mkDeterministicUuid('a', row.AcknowledgementId) + typ := objectTypes[row.ObjecttypeId] + hostId := calcObjectId(env, row.Name1) + serviceId := calcServiceId(env, row.Name1, row.Name2) + + acknowledgementHistoryId := hashAny([]interface{}{ + env, strings.Title(typ), name, float64(utils.UnixMilli(set.Time())), + }) + + if row.AcknowledgementType == 0 { // clear + // The set counterpart should already have been inserted. + acknowledgementHistoryUpdates = append(acknowledgementHistoryUpdates, &AckClear{ + acknowledgementHistoryId, ts, + }) + + h := &history.HistoryAck{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: id}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "ack_clear", + }, + AcknowledgementHistoryId: acknowledgementHistoryId, + SetTime: set, + ClearTime: ts, + } + + h.EventTime.History = h + allHistory = append(allHistory, h) + } else { // set + acknowledgementHistory = append(acknowledgementHistory, &history.AcknowledgementHistory{ + EntityWithoutChecksum: v1.EntityWithoutChecksum{ + IdMeta: v1.IdMeta{Id: acknowledgementHistoryId}, + }, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + SetTime: set, + Author: icingadbTypes.String{NullString: row.AuthorName}, + Comment: icingadbTypes.String{NullString: row.CommentData}, + ExpireTime: convertTime(row.EndTime.Int64, 0), + IsPersistent: icingadbTypes.Bool{Bool: row.PersistentComment != 0, Valid: true}, + IsSticky: icingadbTypes.Bool{Bool: row.IsSticky != 0, Valid: true}, + }) + + h := &history.HistoryAck{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: id}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "ack_set", + }, + AcknowledgementHistoryId: acknowledgementHistoryId, + SetTime: set, + } + + h.EventTime.History = h + allHistory = append(allHistory, h) + } + + checkpoint = row.AcknowledgementId + } + + icingaDbUpdates = [][]interface{}{acknowledgementHistoryUpdates} + icingaDbInserts = [][]interface{}{acknowledgementHistory, allHistory} + return +} + +const commentMigrationQuery = "SELECT ch.commenthistory_id, UNIX_TIMESTAMP(ch.entry_time) entry_time, " + + "ch.entry_time_usec, ch.entry_type, ch.author_name, ch.comment_data, ch.is_persistent, " + + "IFNULL(UNIX_TIMESTAMP(ch.expiration_time), 0) expiration_time, " + + "IFNULL(UNIX_TIMESTAMP(ch.deletion_time), 0) deletion_time, ch.deletion_time_usec, ch.name, " + + "o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + + "FROM icinga_commenthistory ch USE INDEX (PRIMARY) " + + "INNER JOIN icinga_objects o ON o.object_id=ch.object_id " + + "WHERE ch.commenthistory_id > ? " + // where we were interrupted + "ORDER BY ch.commenthistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated + "LIMIT ?" + +type commentRow = struct { + CommenthistoryId uint64 + EntryTime int64 + EntryTimeUsec uint32 + EntryType uint8 + AuthorName string + CommentData string + IsPersistent uint8 + ExpirationTime int64 + DeletionTime int64 + DeletionTimeUsec uint32 + Name string + ObjecttypeId uint8 + Name1 string + Name2 string +} + +func convertCommentRows( + env string, envId, endpointId icingadbTypes.Binary, + _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []commentRow, +) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { + var commentHistory, allHistory []interface{} + for _, row := range idoRows { + id := calcObjectId(env, row.Name) + typ := objectTypes[row.ObjecttypeId] + hostId := calcObjectId(env, row.Name1) + serviceId := calcServiceId(env, row.Name1, row.Name2) + entryTime := convertTime(row.EntryTime, row.EntryTimeUsec) + removeTime := convertTime(row.DeletionTime, row.DeletionTimeUsec) + expireTime := convertTime(row.ExpirationTime, 0) + + commentHistory = append(commentHistory, &history.CommentHistory{ + CommentHistoryEntity: history.CommentHistoryEntity{CommentId: id}, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + CommentHistoryUpserter: history.CommentHistoryUpserter{ + RemoveTime: removeTime, + HasBeenRemoved: icingadbTypes.Bool{Bool: !removeTime.Time().IsZero(), Valid: true}, + }, + EntryTime: entryTime, + Author: row.AuthorName, + Comment: row.CommentData, + EntryType: icingadbTypes.CommentType(row.EntryType), + IsPersistent: icingadbTypes.Bool{Bool: row.IsPersistent != 0, Valid: true}, + IsSticky: icingadbTypes.Bool{Bool: false, Valid: true}, + ExpireTime: expireTime, + }) + + h1 := &history.HistoryComment{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: randomUuid()}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "comment_add", + }, + CommentHistoryId: id, + EntryTime: entryTime, + } + + h1.EventTime.History = h1 + allHistory = append(allHistory, h1) + + if !removeTime.Time().IsZero() { // remove + h2 := &history.HistoryComment{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: randomUuid()}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "comment_remove", + }, + CommentHistoryId: id, + EntryTime: entryTime, + RemoveTime: removeTime, + ExpireTime: expireTime, + } + + h2.EventTime.History = h2 + allHistory = append(allHistory, h2) + } + + checkpoint = row.CommenthistoryId + } + + icingaDbInserts = [][]interface{}{commentHistory, allHistory} + return +} + +const downtimeMigrationQuery = "SELECT dh.downtimehistory_id, UNIX_TIMESTAMP(dh.entry_time) entry_time, " + + "dh.author_name, dh.comment_data, dh.is_fixed, dh.duration, " + + "UNIX_TIMESTAMP(dh.scheduled_start_time) scheduled_start_time, " + + "IFNULL(UNIX_TIMESTAMP(dh.scheduled_end_time), 0) scheduled_end_time, " + + "IFNULL(UNIX_TIMESTAMP(dh.actual_start_time), 0) actual_start_time, dh.actual_start_time_usec, " + + "IFNULL(UNIX_TIMESTAMP(dh.actual_end_time), 0) actual_end_time, dh.actual_end_time_usec, dh.was_cancelled, " + + "IFNULL(UNIX_TIMESTAMP(dh.trigger_time), 0) trigger_time, dh.name, o.objecttype_id, o.name1, " + + "IFNULL(o.name2, '') name2, IFNULL(sd.name, '') triggered_by " + + "FROM icinga_downtimehistory dh USE INDEX (PRIMARY) " + + "INNER JOIN icinga_objects o ON o.object_id=dh.object_id " + + "LEFT JOIN icinga_scheduleddowntime sd ON sd.scheduleddowntime_id=dh.triggered_by_id " + + "WHERE dh.downtimehistory_id > ? " + // where we were interrupted + "ORDER BY dh.downtimehistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated + "LIMIT ?" + +type downtimeRow = struct { + DowntimehistoryId uint64 + EntryTime int64 + AuthorName string + CommentData string + IsFixed uint8 + Duration int64 + ScheduledStartTime int64 + ScheduledEndTime int64 + ActualStartTime int64 + ActualStartTimeUsec uint32 + ActualEndTime int64 + ActualEndTimeUsec uint32 + WasCancelled uint8 + TriggerTime int64 + Name string + ObjecttypeId uint8 + Name1 string + Name2 string + TriggeredBy string +} + +func convertDowntimeRows( + env string, envId, endpointId icingadbTypes.Binary, + _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []downtimeRow, +) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { + var downtimeHistory, allHistory []interface{} + for _, row := range idoRows { + id := calcObjectId(env, row.Name) + typ := objectTypes[row.ObjecttypeId] + hostId := calcObjectId(env, row.Name1) + serviceId := calcServiceId(env, row.Name1, row.Name2) + scheduledStart := convertTime(row.ScheduledStartTime, 0) + scheduledEnd := convertTime(row.ScheduledEndTime, 0) + triggerTime := convertTime(row.TriggerTime, 0) + actualStart := convertTime(row.ActualStartTime, row.ActualStartTimeUsec) + actualEnd := convertTime(row.ActualEndTime, row.ActualEndTimeUsec) + var startTime, endTime, cancelTime icingadbTypes.UnixMilli + + if scheduledEnd.Time().IsZero() { + scheduledEnd = icingadbTypes.UnixMilli(scheduledStart.Time().Add(time.Duration(row.Duration) * time.Second)) + } + + if actualStart.Time().IsZero() { + startTime = scheduledStart + } else { + startTime = actualStart + } + + if actualEnd.Time().IsZero() { + endTime = scheduledEnd + } else { + endTime = actualEnd + } + + if triggerTime.Time().IsZero() { + triggerTime = startTime + } + + if row.WasCancelled != 0 { + cancelTime = actualEnd + } + + downtimeHistory = append(downtimeHistory, &history.DowntimeHistory{ + DowntimeHistoryEntity: history.DowntimeHistoryEntity{DowntimeId: id}, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + DowntimeHistoryUpserter: history.DowntimeHistoryUpserter{ + HasBeenCancelled: icingadbTypes.Bool{Bool: row.WasCancelled != 0, Valid: true}, + CancelTime: cancelTime, + }, + TriggeredById: calcObjectId(env, row.TriggeredBy), + EntryTime: convertTime(row.EntryTime, 0), + Author: row.AuthorName, + Comment: row.CommentData, + IsFlexible: icingadbTypes.Bool{Bool: row.IsFixed == 0, Valid: true}, + FlexibleDuration: uint64(row.Duration) * 1000, + ScheduledStartTime: scheduledStart, + ScheduledEndTime: scheduledEnd, + StartTime: startTime, + EndTime: endTime, + TriggerTime: triggerTime, + }) + + h1 := &history.HistoryDowntime{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: randomUuid()}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "downtime_start", + }, + DowntimeHistoryId: id, + StartTime: startTime, + } + + h1.EventTime.History = h1 + allHistory = append(allHistory, h1) + + if !actualEnd.Time().IsZero() { // remove + h2 := &history.HistoryDowntime{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: randomUuid()}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "downtime_end", + }, + DowntimeHistoryId: id, + StartTime: startTime, + CancelTime: cancelTime, + EndTime: endTime, + HasBeenCancelled: icingadbTypes.Bool{Bool: row.WasCancelled != 0, Valid: true}, + } + + h2.EventTime.History = h2 + allHistory = append(allHistory, h2) + } + + checkpoint = row.DowntimehistoryId + } + + icingaDbInserts = [][]interface{}{downtimeHistory, allHistory} + return +} + +const flappingMigrationQuery = "SELECT fh.flappinghistory_id, UNIX_TIMESTAMP(fh.event_time) event_time, " + + "fh.event_time_usec, fh.event_type, fh.percent_state_change, fh.low_threshold, " + + "fh.high_threshold, o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + + "FROM icinga_flappinghistory fh USE INDEX (PRIMARY) " + + "INNER JOIN icinga_objects o ON o.object_id=fh.object_id " + + "WHERE fh.flappinghistory_id > ? " + // where we were interrupted + "ORDER BY fh.flappinghistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated + "LIMIT ?" + +// FlappingEnd updates an already migrated start event with the end event info. +type FlappingEnd struct { + Id icingadbTypes.Binary + EndTime icingadbTypes.UnixMilli +} + +// Assert interface compliance. +var _ contracts.TableNamer = (*FlappingEnd)(nil) + +// TableName implements the contracts.TableNamer interface. +func (*FlappingEnd) TableName() string { + return "flapping_history" +} + +type flappingRow = struct { + FlappinghistoryId uint64 + EventTime int64 + EventTimeUsec uint32 + EventType uint16 + PercentStateChange sql.NullFloat64 + LowThreshold float64 + HighThreshold float64 + ObjecttypeId uint8 + Name1 string + Name2 string +} + +func convertFlappingRows( + env string, envId, endpointId icingadbTypes.Binary, + selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []flappingRow, +) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) { + if len(idoRows) < 1 { + return + } + + var cached []struct { + HistoryId uint64 + EventTime int64 + EventTimeUsec uint32 + } + selectCache( + &cached, "SELECT history_id, event_time, event_time_usec FROM end_start_time WHERE history_id BETWEEN ? AND ?", + idoRows[0].FlappinghistoryId, idoRows[len(idoRows)-1].FlappinghistoryId, + ) + + // Needed for start time (see below). + cachedById := make(map[uint64]icingadbTypes.UnixMilli, len(cached)) + for _, c := range cached { + cachedById[c.HistoryId] = convertTime(c.EventTime, c.EventTimeUsec) + } + + var flappingHistory, flappingHistoryUpdates, allHistory []interface{} + for _, row := range idoRows { + ts := convertTime(row.EventTime, row.EventTimeUsec) + + // Needed for ID (see below). + var start icingadbTypes.UnixMilli + if row.EventType == 1001 { // end + var ok bool + start, ok = cachedById[row.FlappinghistoryId] + + if !ok { + continue + } + } else { + start = ts + } + + name := row.Name1 + if row.Name2 != "" { + name += "!" + row.Name2 + } + + id := mkDeterministicUuid('f', row.FlappinghistoryId) + typ := objectTypes[row.ObjecttypeId] + hostId := calcObjectId(env, row.Name1) + serviceId := calcServiceId(env, row.Name1, row.Name2) + + flappingHistoryId := hashAny([]interface{}{ + env, strings.Title(typ), name, float64(utils.UnixMilli(start.Time())), + }) + + if row.EventType == 1001 { // end + // The start counterpart should already have been inserted. + flappingHistoryUpdates = append(flappingHistoryUpdates, &FlappingEnd{flappingHistoryId, ts}) + + h := &history.HistoryFlapping{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: id}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "flapping_end", + }, + FlappingHistoryId: flappingHistoryId, + StartTime: start, + EndTime: ts, + } + + h.EventTime.History = h + allHistory = append(allHistory, h) + } else { // end + flappingHistory = append(flappingHistory, &history.FlappingHistory{ + EntityWithoutChecksum: v1.EntityWithoutChecksum{ + IdMeta: v1.IdMeta{Id: flappingHistoryId}, + }, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + FlappingHistoryUpserter: history.FlappingHistoryUpserter{ + FlappingThresholdLow: float32(row.LowThreshold), + FlappingThresholdHigh: float32(row.HighThreshold), + }, + StartTime: start, + PercentStateChangeStart: icingadbTypes.Float{NullFloat64: row.PercentStateChange}, + }) + + h := &history.HistoryFlapping{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: id}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "flapping_start", + }, + FlappingHistoryId: flappingHistoryId, + StartTime: start, + } + + h.EventTime.History = h + allHistory = append(allHistory, h) + } + + checkpoint = row.FlappinghistoryId + } + + icingaDbUpdates = [][]interface{}{flappingHistoryUpdates} + icingaDbInserts = [][]interface{}{flappingHistory, allHistory} + return +} + +const notificationMigrationQuery = "SELECT n.notification_id, n.notification_reason, " + + "UNIX_TIMESTAMP(n.end_time) end_time, n.end_time_usec, n.state, IFNULL(n.output, '') output, " + + "n.long_output, n.contacts_notified, o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + + "FROM icinga_notifications n USE INDEX (PRIMARY) " + + "INNER JOIN icinga_objects o ON o.object_id=n.object_id " + + "WHERE n.notification_id <= ? AND " + + "n.notification_id > ? " + // where we were interrupted + "ORDER BY n.notification_id " + // allows computeProgress() not to check all IDO rows for whether migrated + "LIMIT ?" + +// zeroHash is a NULL alternative for NOT NULL columns. +var zeroHash = make(icingadbTypes.Binary, 20) + +// notificationTypes maps IDO values to Icinga DB ones. +var notificationTypes = map[uint8]icingadbTypes.NotificationType{5: 1, 6: 2, 7: 4, 8: 8, 1: 16, 2: 128, 3: 256} + +type notificationRow = struct { + NotificationId uint64 + NotificationReason uint8 + EndTime int64 + EndTimeUsec uint32 + State uint8 + Output string + LongOutput sql.NullString + ContactsNotified uint16 + ObjecttypeId uint8 + Name1 string + Name2 string +} + +func convertNotificationRows( + env string, envId, endpointId icingadbTypes.Binary, + selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []notificationRow, +) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { + if len(idoRows) < 1 { + return + } + + var cached []struct { + HistoryId uint64 + PreviousHardState uint8 + } + selectCache( + &cached, "SELECT history_id, previous_hard_state FROM previous_hard_state WHERE history_id BETWEEN ? AND ?", + idoRows[0].NotificationId, idoRows[len(idoRows)-1].NotificationId, + ) + + cachedById := make(map[uint64]uint8, len(cached)) + for _, c := range cached { + cachedById[c.HistoryId] = c.PreviousHardState + } + + var contacts []struct { + NotificationId uint64 + Name1 string + } + + { + const query = "SELECT c.notification_id, o.name1 FROM icinga_contactnotifications c " + + "INNER JOIN icinga_objects o ON o.object_id=c.contact_object_id WHERE c.notification_id BETWEEN ? AND ?" + + err := ido.Select(&contacts, query, idoRows[0].NotificationId, idoRows[len(idoRows)-1].NotificationId) + if err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + } + + contactsById := map[uint64]map[string]struct{}{} + for _, contact := range contacts { + perId, ok := contactsById[contact.NotificationId] + if !ok { + perId = map[string]struct{}{} + contactsById[contact.NotificationId] = perId + } + + perId[contact.Name1] = struct{}{} + } + + var notificationHistory, userNotificationHistory, allHistory []interface{} + for _, row := range idoRows { + previousHardState, ok := cachedById[row.NotificationId] + if !ok { + continue + } + + id := mkDeterministicUuid('n', row.NotificationId) + typ := objectTypes[row.ObjecttypeId] + hostId := calcObjectId(env, row.Name1) + serviceId := calcServiceId(env, row.Name1, row.Name2) + ts := convertTime(row.EndTime, row.EndTimeUsec) + + var nt icingadbTypes.NotificationType + if row.NotificationReason == 0 { + if row.State == 0 { + nt = 64 // recovery + } else { + nt = 32 // problem + } + } else { + nt = notificationTypes[row.NotificationReason] + } + + text := row.Output + if row.LongOutput.Valid { + text += "\n\n" + row.LongOutput.String + } + + notificationHistory = append(notificationHistory, &history.NotificationHistory{ + HistoryTableEntity: history.HistoryTableEntity{Id: id}, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + NotificationId: zeroHash, + Type: nt, + SendTime: ts, + State: row.State, + PreviousHardState: previousHardState, + Author: "-", + Text: text, + UsersNotified: row.ContactsNotified, + }) + + allHistory = append(allHistory, &history.HistoryNotification{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: id}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "notification", + }, + NotificationHistoryId: id, + EventTime: ts, + }) + + for contact := range contactsById[row.NotificationId] { + userId := calcObjectId(env, contact) + + userNotificationHistory = append(userNotificationHistory, &history.UserNotificationHistory{ + EntityWithoutChecksum: v1.EntityWithoutChecksum{ + IdMeta: v1.IdMeta{Id: utils.Checksum(append(append([]byte(nil), id.UUID[:]...), userId...))}, + }, + EnvironmentMeta: v1.EnvironmentMeta{EnvironmentId: envId}, + NotificationHistoryId: id, + UserId: userId, + }) + } + + checkpoint = row.NotificationId + } + + icingaDbInserts = [][]interface{}{notificationHistory, userNotificationHistory, allHistory} + return +} + +const stateMigrationQuery = "SELECT sh.statehistory_id, UNIX_TIMESTAMP(sh.state_time) state_time, " + + "sh.state_time_usec, sh.state, sh.state_type, sh.current_check_attempt, " + + "sh.max_check_attempts, sh.last_state, sh.last_hard_state, sh.output, sh.long_output, " + + "sh.check_source, o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + + "FROM icinga_statehistory sh USE INDEX (PRIMARY) " + + "INNER JOIN icinga_objects o ON o.object_id=sh.object_id " + + "WHERE sh.statehistory_id <= ? AND " + + "sh.statehistory_id > ? " + // where we were interrupted + "ORDER BY sh.statehistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated + "LIMIT ?" + +type stateRow = struct { + StatehistoryId uint64 + StateTime int64 + StateTimeUsec uint32 + State uint8 + StateType uint8 + CurrentCheckAttempt uint16 + MaxCheckAttempts uint16 + LastState uint8 + LastHardState uint8 + Output sql.NullString + LongOutput sql.NullString + CheckSource sql.NullString + ObjecttypeId uint8 + Name1 string + Name2 string +} + +func convertStateRows( + env string, envId, endpointId icingadbTypes.Binary, + selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []stateRow, +) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { + if len(idoRows) < 1 { + return + } + + var cached []struct { + HistoryId uint64 + PreviousHardState uint8 + } + selectCache( + &cached, "SELECT history_id, previous_hard_state FROM previous_hard_state WHERE history_id BETWEEN ? AND ?", + idoRows[0].StatehistoryId, idoRows[len(idoRows)-1].StatehistoryId, + ) + + cachedById := make(map[uint64]uint8, len(cached)) + for _, c := range cached { + cachedById[c.HistoryId] = c.PreviousHardState + } + + var stateHistory, allHistory []interface{} + for _, row := range idoRows { + previousHardState, ok := cachedById[row.StatehistoryId] + if !ok { + continue + } + + id := mkDeterministicUuid('s', row.StatehistoryId) + typ := objectTypes[row.ObjecttypeId] + hostId := calcObjectId(env, row.Name1) + serviceId := calcServiceId(env, row.Name1, row.Name2) + ts := convertTime(row.StateTime, row.StateTimeUsec) + + stateHistory = append(stateHistory, &history.StateHistory{ + HistoryTableEntity: history.HistoryTableEntity{Id: id}, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + EventTime: ts, + StateType: icingadbTypes.StateType(row.StateType), + SoftState: row.State, + HardState: row.LastHardState, + PreviousSoftState: row.LastState, + PreviousHardState: previousHardState, + Attempt: uint8(row.CurrentCheckAttempt), + Output: icingadbTypes.String{NullString: row.Output}, + LongOutput: icingadbTypes.String{NullString: row.LongOutput}, + MaxCheckAttempts: uint32(row.MaxCheckAttempts), + CheckSource: icingadbTypes.String{NullString: row.CheckSource}, + }) + + allHistory = append(allHistory, &history.HistoryState{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: id}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "state_change", + }, + StateHistoryId: id, + EventTime: ts, + }) + + checkpoint = row.StatehistoryId + } + + icingaDbInserts = [][]interface{}{stateHistory, allHistory} + return +} diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index e7a03604..cb33a6af 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -3,12 +3,14 @@ package main import ( "bytes" "context" + "crypto/sha1" "database/sql" "fmt" "github.com/goccy/go-yaml" "github.com/icinga/icingadb/cmd/internal" "github.com/icinga/icingadb/pkg/config" "github.com/icinga/icingadb/pkg/icingadb" + icingadbTypes "github.com/icinga/icingadb/pkg/types" "github.com/jessevdk/go-flags" "github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx/reflectx" @@ -18,6 +20,7 @@ import ( "golang.org/x/sync/errgroup" "os" "path" + "reflect" "strings" "time" ) @@ -35,7 +38,8 @@ type Config struct { IDO config.Database `yaml:"ido"` IcingaDB config.Database `yaml:"icingadb"` Icinga2 struct { - Env string `yaml:"env"` + Env string `yaml:"env"` + Endpoint string `yaml:"endpoint"` } `yaml:"icinga2"` } @@ -71,6 +75,9 @@ func run() int { log.Info("Filling cache") fillCache() + log.Info("Actually migrating") + migrate(c, idb) + return internal.ExitSuccess } @@ -206,7 +213,7 @@ func computeProgress(c *Config, idb *icingadb.DB) { } }() - sliceIdoHistory(ht.snapshot, query, 0, func(rows []ProgressRow) (checkpoint interface{}) { + sliceIdoHistory(ht.snapshot, query, nil, 0, func(rows []ProgressRow) (checkpoint interface{}) { if len(rows) != lastRowsLen { if lastStmt != nil { lastStmt.Close() @@ -289,3 +296,154 @@ func fillCache() { progress.Wait() } + +var tAny = reflect.TypeOf((*interface{})(nil)). // *interface{} + Elem() // interface{} + +func migrate(c *Config, idb *icingadb.DB) { + envId := sha1.Sum([]byte(c.Icinga2.Env)) + endpointId := sha1.Sum([]byte(c.Icinga2.Endpoint)) + + progress := mpb.New() + for i := range types { + types[i].setupBar(progress) + } + + types.forEach(func(ht *historyType) { + vConvertRows := reflect.ValueOf(ht.convertRows) // TODO: make historyType#convertRows generic[T] one nice day + + tRows := vConvertRows.Type(). // func(env string, envId, endpointId Binary, selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []T) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) + In(5) // []T + + var lastQuery string + var lastStmt *sqlx.Stmt + + defer func() { + if lastStmt != nil { + _ = lastStmt.Close() + } + }() + + vConvertRowsArgs := [6]reflect.Value{ + reflect.ValueOf(c.Icinga2.Env), reflect.ValueOf(icingadbTypes.Binary(envId[:])), + reflect.ValueOf(icingadbTypes.Binary(endpointId[:])), + reflect.ValueOf(func(dest interface{}, query string, args ...interface{}) { + if query != lastQuery { + if lastStmt != nil { + _ = lastStmt.Close() + } + + var err error + + lastStmt, err = ht.cache.Preparex(query) + if err != nil { + log.With("backend", "cache", "query", query). + Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + + lastQuery = query + } + + if err := lastStmt.Select(dest, args...); err != nil { + log.With("backend", "cache", "query", query, "args", args). + Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + }), + reflect.ValueOf(ht.snapshot), + } + + var args []interface{} + if ht.cacheLimitQuery != "" { + var limit uint64 + cacheGet(ht.cache, &limit, ht.cacheLimitQuery) + args = append(args, limit) + } + + icingaDbInserts := map[reflect.Type]string{} + icingaDbUpdates := map[reflect.Type]string{} + inc := barIncrementer{ht.bar, time.Now()} + + sliceIdoHistory( + ht.snapshot, ht.migrationQuery, args, ht.lastId, + reflect.MakeFunc(reflect.FuncOf([]reflect.Type{tRows}, []reflect.Type{tAny}, false), + func(args []reflect.Value) []reflect.Value { + vConvertRowsArgs[5] = args[0] + res := vConvertRows.Call(vConvertRowsArgs[:]) + + tx, err := idb.Beginx() + if err != nil { + log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) + } + + for _, table := range res[0].Interface().([][]interface{}) { + if len(table) < 1 { + continue + } + + tRow := reflect.TypeOf(table[0]) + + update, ok := icingaDbUpdates[tRow] + if !ok { + update, _ = idb.BuildUpdateStmt(table[0]) + icingaDbUpdates[tRow] = update + } + + stmt, err := tx.PrepareNamed(update) + if err != nil { + log.With("backend", "Icinga DB", "dml", update). + Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) + } + + for _, row := range table { + if _, err := stmt.Exec(row); err != nil { + log.With("backend", "Icinga DB", "dml", update, "args", row). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + + _ = stmt.Close() + } + + for _, table := range res[1].Interface().([][]interface{}) { + if len(table) < 1 { + continue + } + + tRow := reflect.TypeOf(table[0]) + + insert, ok := icingaDbInserts[tRow] + if !ok { + insert, _ = idb.BuildInsertStmt(table[0]) + insert = "REPLACE " + strings.TrimPrefix(insert, "INSERT ") + icingaDbInserts[tRow] = insert + } + + for len(table) > 0 { + slice := table + if len(slice) > 1000 { + slice = slice[:1000] + table = table[1000:] + } else { + table = nil + } + + if _, err := tx.NamedExec(insert, slice); err != nil { + log.With("backend", "Icinga DB", "dml", insert, "args", slice). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + } + + if err := tx.Commit(); err != nil { + log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) + } + + inc.inc(args[0].Len()) + return res[2:] + }, + ).Interface(), + ) + }) + + progress.Wait() +} diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index d6600564..6ef26142 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -1,12 +1,15 @@ package main import ( + "bufio" "bytes" "context" + "crypto/rand" "crypto/sha1" "encoding/binary" "github.com/google/uuid" "github.com/icinga/icingadb/pkg/icingadb/objectpacker" + icingadbTypes "github.com/icinga/icingadb/pkg/types" "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/vbauerster/mpb/v6" @@ -14,6 +17,7 @@ import ( "go.uber.org/zap" "golang.org/x/sync/errgroup" "reflect" + "sync" "time" ) @@ -52,6 +56,8 @@ var log = func() *zap.SugaredLogger { return logger.Sugar() }() +var objectTypes = map[uint8]string{1: "host", 2: "service"} + // mkDeterministicUuid returns a formally random UUID (v4) as follows: 11111122-3300-4455-4455-555555555555 // // 0: zeroed @@ -60,7 +66,7 @@ var log = func() *zap.SugaredLogger { // 3: "h" (for "history") // 4: the new UUID's formal version (unused bits zeroed) // 5: the ID of the row the new UUID is for in the IDO (big endian) -func mkDeterministicUuid(table byte, rowId uint64) []byte { +func mkDeterministicUuid(table byte, rowId uint64) icingadbTypes.UUID { uid := uuidTemplate uid[3] = table @@ -73,7 +79,7 @@ func mkDeterministicUuid(table byte, rowId uint64) []byte { uid[7] = bEId[0] copy(uid[9:], bEId[1:]) - return uid[:] + return icingadbTypes.UUID{UUID: uid} } // uuidTemplate is for mkDeterministicUuid. @@ -90,6 +96,42 @@ var uuidTemplate = func() uuid.UUID { return uid }() +// randomUuid generates a new UUIDv4. +func randomUuid() icingadbTypes.UUID { + var rander *bufio.Reader + + massRanders.Lock() + for r := range massRanders.pool { + rander = r + delete(massRanders.pool, r) + break + } + massRanders.Unlock() + + if rander == nil { + rander = bufio.NewReader(rand.Reader) + } + + id, err := uuid.NewRandomFromReader(rander) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't generate random UUID")) + } + + massRanders.Lock() + massRanders.pool[rander] = struct{}{} + massRanders.Unlock() + + return icingadbTypes.UUID{UUID: id} +} + +var massRanders = struct { + sync.Mutex + pool map[*bufio.Reader]struct{} +}{ + sync.Mutex{}, + map[*bufio.Reader]struct{}{}, +} + // hashAny combines PackAny and SHA1 hashing. func hashAny(in interface{}) []byte { hash := sha1.New() @@ -100,12 +142,34 @@ func hashAny(in interface{}) []byte { return hash.Sum(nil) } +// convertTime converts *nix timestamps from the IDO for Icinga DB. +func convertTime(ts int64, tsUs uint32) icingadbTypes.UnixMilli { + if ts == 0 && tsUs == 0 { + return icingadbTypes.UnixMilli{} + } + + return icingadbTypes.UnixMilli(time.Unix(ts, int64(tsUs)*int64(time.Microsecond/time.Nanosecond))) +} + // calcObjectId calculates the ID of the config object named name1 for Icinga DB. func calcObjectId(env, name1 string) []byte { + if name1 == "" { + return nil + } + return hashAny([2]string{env, name1}) } -func sliceIdoHistory(snapshot *sqlx.Tx, query string, checkpoint, onRows interface{}) { +// calcServiceId calculates the ID of the service name2 of the host name1 for Icinga DB. +func calcServiceId(env, name1, name2 string) []byte { + if name2 == "" { + return nil + } + + return hashAny([2]string{env, name1 + "!" + name2}) +} + +func sliceIdoHistory(snapshot *sqlx.Tx, query string, args []interface{}, checkpoint, onRows interface{}) { vOnRows := reflect.ValueOf(onRows) // TODO: make onRows generic[T] one nice day tRows := vOnRows.Type(). // func(rows []T) (checkpoint interface{}) @@ -116,15 +180,10 @@ func sliceIdoHistory(snapshot *sqlx.Tx, query string, checkpoint, onRows interfa vRows := vNewRows.Elem() onRowsArgs := [1]reflect.Value{vRows} vZeroRows := reflect.Zero(tRows) - - stmt, err := snapshot.Preparex(query) - if err != nil { - log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - defer stmt.Close() + args = append(append([]interface{}(nil), args...), checkpoint, bulk) for { - if err := stmt.Select(rowsPtr, checkpoint, bulk); err != nil { + if err := snapshot.Select(rowsPtr, query, args...); err != nil { log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) } @@ -137,19 +196,23 @@ func sliceIdoHistory(snapshot *sqlx.Tx, query string, checkpoint, onRows interfa } vRows.Set(vZeroRows) + args[len(args)-2] = checkpoint } } type historyType struct { - name string - idoTable string - idoIdColumn string - idoColumns []string - idbTable string - idbIdColumn string - convertId func(row ProgressRow, env string) []byte - cacheSchema []string - cacheFiller func(*historyType) + name string + idoTable string + idoIdColumn string + idoColumns []string + idbTable string + idbIdColumn string + convertId func(row ProgressRow, env string) []byte + cacheSchema []string + cacheFiller func(*historyType) + cacheLimitQuery string + migrationQuery string + convertRows interface{} cache *sqlx.DB snapshot *sqlx.Tx @@ -193,7 +256,7 @@ var types = historyTypes{ nil, "history", "id", - func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('a', row.Id) }, + func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('a', row.Id); return u.UUID[:] }, eventTimeCacheSchema, func(ht *historyType) { buildEventTimeCache(ht, []string{ @@ -201,6 +264,9 @@ var types = historyTypes{ "xh.entry_time_usec event_time_usec", "xh.acknowledgement_type event_is_start", "xh.object_id", }) }, + "", + acknowledgementMigrationQuery, + convertAcknowledgementRows, nil, nil, 0, nil, 0, }, { @@ -211,7 +277,10 @@ var types = historyTypes{ "comment_history", "comment_id", func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, - nil, nil, nil, nil, 0, nil, 0, + nil, nil, "", + commentMigrationQuery, + convertCommentRows, + nil, nil, 0, nil, 0, }, { "downtime", @@ -221,7 +290,10 @@ var types = historyTypes{ "downtime_history", "downtime_id", func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, - nil, nil, nil, nil, 0, nil, 0, + nil, nil, "", + downtimeMigrationQuery, + convertDowntimeRows, + nil, nil, 0, nil, 0, }, { "flapping", @@ -230,7 +302,7 @@ var types = historyTypes{ nil, "history", "id", - func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('f', row.Id) }, + func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('f', row.Id); return u.UUID[:] }, eventTimeCacheSchema, func(ht *historyType) { buildEventTimeCache(ht, []string{ @@ -238,6 +310,9 @@ var types = historyTypes{ "xh.event_time_usec", "xh.event_type-1000 event_is_start", "xh.object_id", }) }, + "", + flappingMigrationQuery, + convertFlappingRows, nil, nil, 0, nil, 0, }, { @@ -247,13 +322,16 @@ var types = historyTypes{ nil, "notification_history", "id", - func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('n', row.Id) }, + func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('n', row.Id); return u.UUID[:] }, previousHardStateCacheSchema, func(ht *historyType) { buildPreviousHardStateCache(ht, []string{ "xh.notification_id id", "xh.object_id", "xh.state last_hard_state", }) }, + "SELECT MAX(history_id) FROM previous_hard_state", + notificationMigrationQuery, + convertNotificationRows, nil, nil, 0, nil, 0, }, { @@ -263,11 +341,14 @@ var types = historyTypes{ nil, "state_history", "id", - func(row ProgressRow, _ string) []byte { return mkDeterministicUuid('s', row.Id) }, + func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('s', row.Id); return u.UUID[:] }, previousHardStateCacheSchema, func(ht *historyType) { buildPreviousHardStateCache(ht, []string{"xh.statehistory_id id", "xh.object_id", "xh.last_hard_state"}) }, + "SELECT MAX(history_id) FROM previous_hard_state", + stateMigrationQuery, + convertStateRows, nil, nil, 0, nil, 0, }, } diff --git a/cmd/ido2icingadb/misc_test.go b/cmd/ido2icingadb/misc_test.go index 76c5aec4..e3266e88 100644 --- a/cmd/ido2icingadb/misc_test.go +++ b/cmd/ido2icingadb/misc_test.go @@ -6,10 +6,8 @@ import ( ) func TestMkDeterministicUuid(t *testing.T) { - if !bytes.Equal( - mkDeterministicUuid('s', 0x0102030405060708), - []byte{'I', 'D', 'O', 's', 'h', 0, 0x40, 1, 0x80, 2, 3, 4, 5, 6, 7, 8}, - ) { + id := mkDeterministicUuid('s', 0x0102030405060708).UUID + if !bytes.Equal(id[:], []byte{'I', 'D', 'O', 's', 'h', 0, 0x40, 1, 0x80, 2, 3, 4, 5, 6, 7, 8}) { t.Error("got wrong UUID from mkDeterministicUuid(stateHistory, 0x0102030405060708)") } } From 517fdcb4752c2e1a16ad748cfb787c46cd19e9bb Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 30 Sep 2021 15:28:31 +0200 Subject: [PATCH 021/103] cmd/ido2icingadb: correct log levels --- cmd/ido2icingadb/cache.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 82cc4acf..dfef5231 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -292,14 +292,14 @@ func cacheGet(cache interface { }, dest interface{}, query string, args ...interface{}) { if err := cache.Get(dest, query, args...); err != nil { log.With("backend", "cache", "query", query, "args", args). - Errorf("%+v", errors.Wrap(err, "can't perform query")) + Fatalf("%+v", errors.Wrap(err, "can't perform query")) } } func cacheSelect(cacheTx *sqlx.Tx, dest interface{}, query string, args ...interface{}) { if err := cacheTx.Select(dest, query, args...); err != nil { log.With("backend", "cache", "query", query, "args", args). - Errorf("%+v", errors.Wrap(err, "can't perform query")) + Fatalf("%+v", errors.Wrap(err, "can't perform query")) } } From ae57d4fe6ff259dedd120b242d819e254838d7e8 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 30 Sep 2021 15:29:03 +0200 Subject: [PATCH 022/103] cmd/ido2icingadb: reduce IDE warnings --- cmd/ido2icingadb/main.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index cb33a6af..45aeb548 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -58,7 +58,7 @@ func run() int { return ex } - defer log.Sync() + defer func() { _ = log.Sync() }() log.Info("Starting IDO to Icinga DB history migration") @@ -69,7 +69,7 @@ func run() int { log.Info("Computing progress") countIdoHistory() - log.Sync() + _ = log.Sync() computeProgress(c, idb) log.Info("Filling cache") @@ -84,14 +84,14 @@ func run() int { func parseConfig(f *Flags) (*Config, int) { cf, err := os.Open(f.Config) if err != nil { - fmt.Fprintf(os.Stderr, "can't open config file: %s\n", err.Error()) + _, _ = fmt.Fprintf(os.Stderr, "can't open config file: %s\n", err.Error()) return nil, 2 } - defer cf.Close() + defer func() { _ = cf.Close() }() c := &Config{} if err := yaml.NewDecoder(cf).Decode(c); err != nil { - fmt.Fprintf(os.Stderr, "can't parse config file: %s\n", err.Error()) + _, _ = fmt.Fprintf(os.Stderr, "can't parse config file: %s\n", err.Error()) return nil, 2 } @@ -209,18 +209,27 @@ func computeProgress(c *Config, idb *icingadb.DB) { defer func() { if lastStmt != nil { - lastStmt.Close() + _ = lastStmt.Close() } }() sliceIdoHistory(ht.snapshot, query, nil, 0, func(rows []ProgressRow) (checkpoint interface{}) { if len(rows) != lastRowsLen { if lastStmt != nil { - lastStmt.Close() + _ = lastStmt.Close() } buf := &bytes.Buffer{} - fmt.Fprintf(buf, "SELECT %s FROM %s WHERE %s IN (?", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn) + + { + _, err := fmt.Fprintf( + buf, "SELECT %s FROM %s WHERE %s IN (?", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn, + ) + if err != nil { + // programming error + panic(err) + } + } for i := 1; i < len(rows); i++ { buf.Write([]byte(",?")) From d9e2c7daf43ea69f56fb6cd67bde7e2ba9f5de19 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Oct 2021 15:19:21 +0200 Subject: [PATCH 023/103] cmd/ido2icingadb: reason not actually used INNER JOINs --- cmd/ido2icingadb/cache.go | 4 ++++ cmd/ido2icingadb/main.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index dfef5231..c6e24409 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -46,6 +46,8 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { sliceIdoHistory( ht.snapshot, "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ + // For actual migration icinga_objects will be joined anyway, + // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" > ? ORDER BY xh."+ht.idoIdColumn+" LIMIT ?", nil, checkpoint.MaxId.Int64, @@ -171,6 +173,8 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { sliceIdoHistory( ht.snapshot, "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ + // For actual migration icinga_objects will be joined anyway, + // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" < ? ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT ?", nil, checkpoint, diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 45aeb548..fafd51cd 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -175,6 +175,8 @@ func countIdoHistory() { types.forEach(func(ht *historyType) { err := ht.snapshot.Get( &ht.total, + // For actual migration icinga_objects will be joined anyway, + // so it makes no sense to take vanished objects into account. "SELECT COUNT(*) FROM "+ht.idoTable+" xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id", ) if err != nil { @@ -199,6 +201,8 @@ func computeProgress(c *Config, idb *icingadb.DB) { query := "SELECT xh." + strings.Join(append(append([]string(nil), ht.idoColumns...), ht.idoIdColumn), ", xh.") + " id FROM " + + // For actual migration icinga_objects will be joined anyway, + // so it makes no sense to take vanished objects into account. ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + ht.idoIdColumn + " > ? ORDER BY xh." + ht.idoIdColumn + " LIMIT ?" From 87a6f8ede7f04d416cd037de14c754db28ff12ce Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Oct 2021 15:20:53 +0200 Subject: [PATCH 024/103] cmd/ido2icingadb: shorten SugaredLogger#With().Info() to #Infow() --- cmd/ido2icingadb/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index fafd51cd..a152682e 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -183,7 +183,7 @@ func countIdoHistory() { log.Fatalf("%+v", errors.Wrap(err, "can't count query")) } - log.With("type", ht.name, "amount", ht.total).Info("Counted total IDO events") + log.Infow("Counted total IDO events", "type", ht.name, "amount", ht.total) }) } From 21115334c4bc24a0bb017644c69dfd179c692b4d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Oct 2021 15:44:18 +0200 Subject: [PATCH 025/103] cmd/ido2icingadb: document sliceIdoHistory() --- cmd/ido2icingadb/misc.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 6ef26142..a0c18036 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -169,6 +169,11 @@ func calcServiceId(env, name1, name2 string) []byte { return hashAny([2]string{env, name1 + "!" + name2}) } +// sliceIdoHistory performs query with args+[]interface{}{checkpoint,bulk} on snapshot and passes the results +// to onRows (a func([]T)interface{}) until either an empty result set or onRows() returns nil. +// Rationale: split the likely large result set of a query by adding a WHERE condition and a LIMIT, +// both with ? placeholders. Due to this function's internals they have to be the last two placeholders. +// checkpoint is the initial value for the WHERE condition, onRows() returns follow-up ones. func sliceIdoHistory(snapshot *sqlx.Tx, query string, args []interface{}, checkpoint, onRows interface{}) { vOnRows := reflect.ValueOf(onRows) // TODO: make onRows generic[T] one nice day From 31e7c384047574374b661b435d9fbf6ed11f1afd Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Oct 2021 15:47:32 +0200 Subject: [PATCH 026/103] cmd/ido2icingadb: types: add keys and delete zero values --- cmd/ido2icingadb/misc.go | 134 ++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 74 deletions(-) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index a0c18036..c7d7c2a7 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -255,105 +255,91 @@ func (ht *historyTypes) forEach(f func(*historyType)) { var types = historyTypes{ { - "acknowledgement", - "icinga_acknowledgements", - "acknowledgement_id", - nil, - "history", - "id", - func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('a', row.Id); return u.UUID[:] }, - eventTimeCacheSchema, - func(ht *historyType) { + name: "acknowledgement", + idoTable: "icinga_acknowledgements", + idoIdColumn: "acknowledgement_id", + idbTable: "history", + idbIdColumn: "id", + convertId: func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('a', row.Id); return u.UUID[:] }, + cacheSchema: eventTimeCacheSchema, + cacheFiller: func(ht *historyType) { buildEventTimeCache(ht, []string{ "xh.acknowledgement_id id", "UNIX_TIMESTAMP(xh.entry_time) event_time", "xh.entry_time_usec event_time_usec", "xh.acknowledgement_type event_is_start", "xh.object_id", }) }, - "", - acknowledgementMigrationQuery, - convertAcknowledgementRows, - nil, nil, 0, nil, 0, + migrationQuery: acknowledgementMigrationQuery, + convertRows: convertAcknowledgementRows, }, { - "comment", - "icinga_commenthistory", - "commenthistory_id", - []string{"name"}, - "comment_history", - "comment_id", - func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, - nil, nil, "", - commentMigrationQuery, - convertCommentRows, - nil, nil, 0, nil, 0, + name: "comment", + idoTable: "icinga_commenthistory", + idoIdColumn: "commenthistory_id", + idoColumns: []string{"name"}, + idbTable: "comment_history", + idbIdColumn: "comment_id", + convertId: func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, + migrationQuery: commentMigrationQuery, + convertRows: convertCommentRows, }, { - "downtime", - "icinga_downtimehistory", - "downtimehistory_id", - []string{"name"}, - "downtime_history", - "downtime_id", - func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, - nil, nil, "", - downtimeMigrationQuery, - convertDowntimeRows, - nil, nil, 0, nil, 0, + name: "downtime", + idoTable: "icinga_downtimehistory", + idoIdColumn: "downtimehistory_id", + idoColumns: []string{"name"}, + idbTable: "downtime_history", + idbIdColumn: "downtime_id", + convertId: func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, + migrationQuery: downtimeMigrationQuery, + convertRows: convertDowntimeRows, }, { - "flapping", - "icinga_flappinghistory", - "flappinghistory_id", - nil, - "history", - "id", - func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('f', row.Id); return u.UUID[:] }, - eventTimeCacheSchema, - func(ht *historyType) { + name: "flapping", + idoTable: "icinga_flappinghistory", + idoIdColumn: "flappinghistory_id", + idbTable: "history", + idbIdColumn: "id", + convertId: func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('f', row.Id); return u.UUID[:] }, + cacheSchema: eventTimeCacheSchema, + cacheFiller: func(ht *historyType) { buildEventTimeCache(ht, []string{ "xh.flappinghistory_id id", "UNIX_TIMESTAMP(xh.event_time) event_time", "xh.event_time_usec", "xh.event_type-1000 event_is_start", "xh.object_id", }) }, - "", - flappingMigrationQuery, - convertFlappingRows, - nil, nil, 0, nil, 0, + migrationQuery: flappingMigrationQuery, + convertRows: convertFlappingRows, }, { - "notification", - "icinga_notifications", - "notification_id", - nil, - "notification_history", - "id", - func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('n', row.Id); return u.UUID[:] }, - previousHardStateCacheSchema, - func(ht *historyType) { + name: "notification", + idoTable: "icinga_notifications", + idoIdColumn: "notification_id", + idbTable: "notification_history", + idbIdColumn: "id", + convertId: func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('n', row.Id); return u.UUID[:] }, + cacheSchema: previousHardStateCacheSchema, + cacheFiller: func(ht *historyType) { buildPreviousHardStateCache(ht, []string{ "xh.notification_id id", "xh.object_id", "xh.state last_hard_state", }) }, - "SELECT MAX(history_id) FROM previous_hard_state", - notificationMigrationQuery, - convertNotificationRows, - nil, nil, 0, nil, 0, + cacheLimitQuery: "SELECT MAX(history_id) FROM previous_hard_state", + migrationQuery: notificationMigrationQuery, + convertRows: convertNotificationRows, }, { - "state", - "icinga_statehistory", - "statehistory_id", - nil, - "state_history", - "id", - func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('s', row.Id); return u.UUID[:] }, - previousHardStateCacheSchema, - func(ht *historyType) { + name: "state", + idoTable: "icinga_statehistory", + idoIdColumn: "statehistory_id", + idbTable: "state_history", + idbIdColumn: "id", + convertId: func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('s', row.Id); return u.UUID[:] }, + cacheSchema: previousHardStateCacheSchema, + cacheFiller: func(ht *historyType) { buildPreviousHardStateCache(ht, []string{"xh.statehistory_id id", "xh.object_id", "xh.last_hard_state"}) }, - "SELECT MAX(history_id) FROM previous_hard_state", - stateMigrationQuery, - convertStateRows, - nil, nil, 0, nil, 0, + cacheLimitQuery: "SELECT MAX(history_id) FROM previous_hard_state", + migrationQuery: stateMigrationQuery, + convertRows: convertStateRows, }, } From 7d2ea6c8b5258ec1afa7a57d4f3f05e841862d97 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Oct 2021 16:16:02 +0200 Subject: [PATCH 027/103] cmd/ido2icingadb: don't re-invent sqlx.In() --- cmd/ido2icingadb/main.go | 59 +++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index a152682e..0da71d57 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "context" "crypto/sha1" "database/sql" @@ -206,6 +205,7 @@ func computeProgress(c *Config, idb *icingadb.DB) { ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + ht.idoIdColumn + " > ? ORDER BY xh." + ht.idoIdColumn + " LIMIT ?" + baseIdbQuery := fmt.Sprintf("SELECT %s FROM %s WHERE %s IN (?)", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn) var lastRowsLen int var lastQuery string var lastStmt *sqlx.Stmt @@ -218,39 +218,6 @@ func computeProgress(c *Config, idb *icingadb.DB) { }() sliceIdoHistory(ht.snapshot, query, nil, 0, func(rows []ProgressRow) (checkpoint interface{}) { - if len(rows) != lastRowsLen { - if lastStmt != nil { - _ = lastStmt.Close() - } - - buf := &bytes.Buffer{} - - { - _, err := fmt.Fprintf( - buf, "SELECT %s FROM %s WHERE %s IN (?", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn, - ) - if err != nil { - // programming error - panic(err) - } - } - - for i := 1; i < len(rows); i++ { - buf.Write([]byte(",?")) - } - - buf.Write([]byte(")")) - lastRowsLen = len(rows) - lastQuery = buf.String() - - var err error - lastStmt, err = idb.Preparex(lastQuery) - - if err != nil { - log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - } - ids := make([]interface{}, 0, len(rows)) converted := make([]convertedId, 0, len(rows)) @@ -260,6 +227,30 @@ func computeProgress(c *Config, idb *icingadb.DB) { converted = append(converted, convertedId{row.Id, conv}) } + if len(rows) != lastRowsLen { + if lastStmt != nil { + _ = lastStmt.Close() + } + + lastRowsLen = len(rows) + + { + var err error + lastQuery, _, err = sqlx.In(baseIdbQuery, ids...) + + if err != nil { + log.With("query", baseIdbQuery).Fatalf("%+v", errors.Wrap(err, "can't assemble query")) + } + } + + var err error + lastStmt, err = idb.Preparex(lastQuery) + + if err != nil { + log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + } + var present [][]byte if err := lastStmt.Select(&present, ids...); err != nil { log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't perform query")) From f4a2741f008daa338a722d99f1b225bbbbd367c8 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 6 Oct 2021 11:36:10 +0200 Subject: [PATCH 028/103] cmd/ido2icingadb: remove redundant function --- cmd/ido2icingadb/main.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 0da71d57..8269cad1 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -6,7 +6,6 @@ import ( "database/sql" "fmt" "github.com/goccy/go-yaml" - "github.com/icinga/icingadb/cmd/internal" "github.com/icinga/icingadb/pkg/config" "github.com/icinga/icingadb/pkg/icingadb" icingadbTypes "github.com/icinga/icingadb/pkg/types" @@ -43,18 +42,14 @@ type Config struct { } func main() { - os.Exit(run()) -} - -func run() int { f := &Flags{} if _, err := flags.NewParser(f, flags.Default).Parse(); err != nil { - return 2 + os.Exit(2) } c, ex := parseConfig(f) if c == nil { - return ex + os.Exit(ex) } defer func() { _ = log.Sync() }() @@ -76,8 +71,6 @@ func run() int { log.Info("Actually migrating") migrate(c, idb) - - return internal.ExitSuccess } func parseConfig(f *Flags) (*Config, int) { From f63892ea014d8cd661933f83155b2cff7f277b0f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Oct 2021 17:14:24 +0200 Subject: [PATCH 029/103] Revert "Outsource cmd/icingadb.Exit* to cmd/internal" This reverts commit fb02ea67ad17af35e72434e97b3e611434405392. --- cmd/icingadb/main.go | 7 ++++--- cmd/internal/common.go | 6 ------ 2 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 cmd/internal/common.go diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go index fb8bc7d9..d778c412 100644 --- a/cmd/icingadb/main.go +++ b/cmd/icingadb/main.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "github.com/go-redis/redis/v8" - "github.com/icinga/icingadb/cmd/internal" "github.com/icinga/icingadb/internal/command" "github.com/icinga/icingadb/pkg/common" "github.com/icinga/icingadb/pkg/driver" @@ -30,6 +29,8 @@ import ( ) const ( + ExitSuccess = 0 + ExitFailure = 1 expectedRedisSchemaVersion = "5" expectedMysqlSchemaVersion = 3 expectedPostgresSchemaVersion = 1 @@ -342,14 +343,14 @@ func run() int { } cancelHactx() - return internal.ExitFailure + return ExitFailure case <-ctx.Done(): logger.Fatalf("%+v", errors.New("main context closed unexpectedly")) case s := <-sig: logger.Infow("Exiting due to signal", zap.String("signal", s.String())) cancelHactx() - return internal.ExitSuccess + return ExitSuccess } } diff --git a/cmd/internal/common.go b/cmd/internal/common.go deleted file mode 100644 index 0aa96924..00000000 --- a/cmd/internal/common.go +++ /dev/null @@ -1,6 +0,0 @@ -package internal - -const ( - ExitSuccess = 0 - ExitFailure = 1 -) From 3ca0b493af5c05bc3968a0ae4a021f11c56b108c Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 6 Oct 2021 12:32:44 +0200 Subject: [PATCH 030/103] cmd/ido2icingadb: document main.go --- cmd/ido2icingadb/main.go | 79 +++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 9 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 8269cad1..f4164bbd 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -35,12 +35,17 @@ type Flags struct { type Config struct { IDO config.Database `yaml:"ido"` IcingaDB config.Database `yaml:"icingadb"` - Icinga2 struct { - Env string `yaml:"env"` + // Icinga2 specifies information the IDO doesn't provide. + Icinga2 struct { + // Env specifies the "Environment" config constant value (likely ""). + Env string `yaml:"env"` + // Endpoint specifies the name on the main endpoint writing to IDO. Endpoint string `yaml:"endpoint"` } `yaml:"icinga2"` } +// main validates the CLI, parses the config and migrates history from IDO to Icinga DB (see comments below). +// Most of the called functions exit the whole program by themselves on non-recoverable errors. func main() { f := &Flags{} if _, err := flags.NewParser(f, flags.Default).Parse(); err != nil { @@ -57,15 +62,28 @@ func main() { log.Info("Starting IDO to Icinga DB history migration") ido, idb := connectAll(c) + + // Start repeatable-read-isolated transactions (consistent SELECTs) + // not to have to care for IDO data changes during migration. startIdoTx(ido) + + // Prepare the directory structure the following fillCache() will need later. mkCache(f, idb.Mapper) log.Info("Computing progress") + // Count total source data, so the following computeProgress() + // knows how many work is left and can display progress bars. countIdoHistory() + + // Roll out a red carpet for the following computeProgress()' progress bars. _ = log.Sync() + + // computeProgress figures out which data has already been migrated + // not to start from the beginning every time in the following migrate(). computeProgress(c, idb) + // On rationale read buildEventTimeCache() and buildPreviousHardStateCache() docs. log.Info("Filling cache") fillCache() @@ -73,7 +91,8 @@ func main() { migrate(c, idb) } -func parseConfig(f *Flags) (*Config, int) { +// parseConfig validates the f.Config file and returns the config and -1 or - on failure - nil and an exit code. +func parseConfig(f *Flags) (_ *Config, exit int) { cf, err := os.Open(f.Config) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "can't open config file: %s\n", err.Error()) @@ -90,6 +109,8 @@ func parseConfig(f *Flags) (*Config, int) { return c, -1 } +// mkCache ensures /.sqlite3 files are present and contain their schema +// and initializes types[*].cache. (On non-recoverable errors the whole program exits.) func mkCache(f *Flags, mapper *reflectx.Mapper) { log.Info("Preparing cache") @@ -121,6 +142,7 @@ func mkCache(f *Flags, mapper *reflectx.Mapper) { }) } +// connectAll connects to ido and idb (Icinga DB) as c specifies. (On non-recoverable errors the whole program exits.) func connectAll(c *Config) (ido, idb *icingadb.DB) { log.Info("Connecting to databases") eg, _ := errgroup.WithContext(context.Background()) @@ -139,6 +161,7 @@ func connectAll(c *Config) (ido, idb *icingadb.DB) { return } +// connect connects to which DB as cfg specifies. (On non-recoverable errors the whole program exits.) func connect(which string, cfg *config.Database) *icingadb.DB { db, err := cfg.Open(log) if err != nil { @@ -152,6 +175,8 @@ func connect(which string, cfg *config.Database) *icingadb.DB { return db } +// startIdoTx initializes types[*].snapshot with new repeatable-read-isolated ido transactions. +// (On non-recoverable errors the whole program exits.) func startIdoTx(ido *icingadb.DB) { types.forEach(func(ht *historyType) { tx, err := ido.BeginTxx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) @@ -163,6 +188,8 @@ func startIdoTx(ido *icingadb.DB) { }) } +// countIdoHistory initializes types[*].total with how many events to migrate. +// (On non-recoverable errors the whole program exits.) func countIdoHistory() { types.forEach(func(ht *historyType) { err := ht.snapshot.Get( @@ -179,6 +206,8 @@ func countIdoHistory() { }) } +// computeProgress initializes types[*].lastId with how many events have already been migrated to idb. +// (On non-recoverable errors the whole program exits.) func computeProgress(c *Config, idb *icingadb.DB) { progress := mpb.New() for i := range types { @@ -196,12 +225,17 @@ func computeProgress(c *Config, idb *icingadb.DB) { // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + - ht.idoIdColumn + " > ? ORDER BY xh." + ht.idoIdColumn + " LIMIT ?" + ht.idoIdColumn + " > ? ORDER BY xh." + // requires migrate() to migrate serially, in order + ht.idoIdColumn + " LIMIT ?" - baseIdbQuery := fmt.Sprintf("SELECT %s FROM %s WHERE %s IN (?)", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn) + // As long as the current chunk is lastRowsLen long (doesn't change)... var lastRowsLen int + + // ... we can re-use these: var lastQuery string var lastStmt *sqlx.Stmt + + baseIdbQuery := fmt.Sprintf("SELECT %s FROM %s WHERE %s IN (?)", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn) inc := barIncrementer{ht.bar, time.Now()} defer func() { @@ -210,16 +244,19 @@ func computeProgress(c *Config, idb *icingadb.DB) { } }() + // Stream IDO IDs, ... sliceIdoHistory(ht.snapshot, query, nil, 0, func(rows []ProgressRow) (checkpoint interface{}) { ids := make([]interface{}, 0, len(rows)) converted := make([]convertedId, 0, len(rows)) + // ... convert them to Icinga DB ones, ... for _, row := range rows { conv := ht.convertId(row, c.Icinga2.Env) ids = append(ids, conv) converted = append(converted, convertedId{row.Id, conv}) } + // ... prepare a new query if the last one doesn't fit, ... if len(rows) != lastRowsLen { if lastStmt != nil { _ = lastStmt.Close() @@ -244,6 +281,7 @@ func computeProgress(c *Config, idb *icingadb.DB) { } } + // ... select which have already been migrated, ... var present [][]byte if err := lastStmt.Select(&present, ids...); err != nil { log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't perform query")) @@ -256,14 +294,17 @@ func computeProgress(c *Config, idb *icingadb.DB) { presentSet[key] = struct{}{} } + // ... and in IDO ID order: for _, conv := range converted { var key [20]byte copy(key[:], conv.idb) + // Stop on the first not yet migrated. if _, ok := presentSet[key]; !ok { return nil } + // If an ID has already been migrated, increase the actual migration's start point. ht.lastId = conv.ido } @@ -277,6 +318,7 @@ func computeProgress(c *Config, idb *icingadb.DB) { progress.Wait() } +// fillCache fills /.sqlite3 (actually types[*].cacheFiller does). func fillCache() { progress := mpb.New() for i := range types { @@ -294,9 +336,11 @@ func fillCache() { progress.Wait() } +// tAny can't be created by just using reflect.TypeOf(interface{}(nil)). var tAny = reflect.TypeOf((*interface{})(nil)). // *interface{} Elem() // interface{} +// migrate does the actual migration. func migrate(c *Config, idb *icingadb.DB) { envId := sha1.Sum([]byte(c.Icinga2.Env)) endpointId := sha1.Sum([]byte(c.Icinga2.Endpoint)) @@ -307,10 +351,15 @@ func migrate(c *Config, idb *icingadb.DB) { } types.forEach(func(ht *historyType) { + // type rowStructPtr = interface{} + // type table = []rowStructPtr + // func convertRowsFromIdoToIcingaDb(env string, envId, endpointId Binary, + // selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []T) + // (icingaDbUpdates, icingaDbInserts []table, checkpoint interface{}) vConvertRows := reflect.ValueOf(ht.convertRows) // TODO: make historyType#convertRows generic[T] one nice day - tRows := vConvertRows.Type(). // func(env string, envId, endpointId Binary, selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []T) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) - In(5) // []T + // []T (idoRows) + tRows := vConvertRows.Type().In(5) var lastQuery string var lastStmt *sqlx.Stmt @@ -324,7 +373,8 @@ func migrate(c *Config, idb *icingadb.DB) { vConvertRowsArgs := [6]reflect.Value{ reflect.ValueOf(c.Icinga2.Env), reflect.ValueOf(icingadbTypes.Binary(envId[:])), reflect.ValueOf(icingadbTypes.Binary(endpointId[:])), - reflect.ValueOf(func(dest interface{}, query string, args ...interface{}) { + reflect.ValueOf(func(dest interface{}, query string, args ...interface{}) { // selectCache + // Prepare new one, if old one doesn't fit anymore. if query != lastQuery { if lastStmt != nil { _ = lastStmt.Close() @@ -347,9 +397,13 @@ func migrate(c *Config, idb *icingadb.DB) { } }), reflect.ValueOf(ht.snapshot), + // and the rows (below) } var args []interface{} + + // For the case that the cache was older that the IDO, + // but ht.cacheFiller couldn't update it, limit (WHERE) our source data set. if ht.cacheLimitQuery != "" { var limit uint64 cacheGet(ht.cache, &limit, ht.cacheLimitQuery) @@ -360,13 +414,18 @@ func migrate(c *Config, idb *icingadb.DB) { icingaDbUpdates := map[reflect.Type]string{} inc := barIncrementer{ht.bar, time.Now()} + // Stream IDO rows, ... sliceIdoHistory( ht.snapshot, ht.migrationQuery, args, ht.lastId, reflect.MakeFunc(reflect.FuncOf([]reflect.Type{tRows}, []reflect.Type{tAny}, false), func(args []reflect.Value) []reflect.Value { - vConvertRowsArgs[5] = args[0] + vConvertRowsArgs[5] = args[0] // pass the rows + + // ... convert them, ... res := vConvertRows.Call(vConvertRowsArgs[:]) + // ... and insert them: + tx, err := idb.Beginx() if err != nil { log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) @@ -391,6 +450,7 @@ func migrate(c *Config, idb *icingadb.DB) { Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) } + // UPDATE à one. for _, row := range table { if _, err := stmt.Exec(row); err != nil { log.With("backend", "Icinga DB", "dml", update, "args", row). @@ -416,6 +476,7 @@ func migrate(c *Config, idb *icingadb.DB) { } for len(table) > 0 { + // REPLACE à 1000. slice := table if len(slice) > 1000 { slice = slice[:1000] From 7e0d3d59a5912f929bd8c7012a35078126cd911d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Oct 2021 12:38:06 +0200 Subject: [PATCH 031/103] cmd/ido2icingadb: document cache.go --- cmd/ido2icingadb/cache.go | 68 +++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index c6e24409..b4e9bff9 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -31,7 +31,7 @@ var eventTimeCacheSchema = []string{ // Therefore: Stream IDO's icinga_flappinghistory once, compute flapping_history#start_time // and cache it into an SQLite database. Then steam from that database and the IDO. // -// Similar for acknowledgements. +// Similar for acknowledgements. (On non-recoverable errors the whole program exits.) func buildEventTimeCache(ht *historyType, idoColumns []string) { chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func()) { var checkpoint struct { @@ -43,6 +43,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { ht.bar.SetCurrent(checkpoint.Cnt * 2) inc := barIncrementer{ht.bar, time.Now()} + // Stream source data... sliceIdoHistory( ht.snapshot, "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ @@ -50,7 +51,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" > ? ORDER BY xh."+ht.idoIdColumn+" LIMIT ?", - nil, checkpoint.MaxId.Int64, + nil, checkpoint.MaxId.Int64, // ... since we were interrupted: func(idoRows []struct { Id uint64 EventTime int64 @@ -60,6 +61,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { }) (checkpoint interface{}) { for _, idoRow := range idoRows { if idoRow.EventIsStart == 0 { + // Ack/flapping end event. Get the start event time: var lst []struct { EventTime int64 EventTimeUsec uint32 @@ -69,22 +71,28 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { idoRow.ObjectId, ) + // If we have that, ... if len(lst) > 0 { + // ... save the start event time for the actual migration: cacheExec( *tx, false, "INSERT INTO end_start_time(history_id, event_time, event_time_usec) VALUES (?, ?, ?)", idoRow.Id, lst[0].EventTime, lst[0].EventTimeUsec, ) + // This previously queried info isn't needed anymore. onDeleted(cacheExec( *tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId, )) } } else { + // Ack/flapping start event directly after another start event (per checkable). + // The old one won't have (but the new one will) an end event (which will need its time). onDeleted(cacheExec( *tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId, )) + // An ack/flapping start event. The following end event (per checkable) will need its time. cacheExec( *tx, false, "INSERT INTO last_start_time(object_id, event_time, event_time_usec) VALUES (?, ?, ?)", @@ -101,6 +109,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { }, ) + // This never queried info isn't needed anymore. onDeleted(cacheExec(*tx, false, "DELETE FROM last_start_time")) }) @@ -134,7 +143,7 @@ var previousHardStateCacheSchema = []string{ // Therefore: Stream IDO's icinga_statehistory once, compute state_history#previous_hard_state // and cache it into an SQLite database. Then steam from that database and the IDO. // -// Similar for notifications. +// Similar for notifications. (On non-recoverable errors the whole program exits.) func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func()) { var nextIds struct { @@ -147,16 +156,16 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { cacheGet(*tx, &previousHardState, "SELECT COUNT(*) cnt FROM previous_hard_state") var checkpoint int64 - if nextIds.MinId.Valid { - checkpoint = nextIds.MinId.Int64 - } else { + if nextIds.MinId.Valid { // there are next_ids + checkpoint = nextIds.MinId.Int64 // this kind of caches is filled descending + } else { // there aren't any next_ids // next_ids contains the most recently processed IDs and is only empty if... if previousHardState.Cnt == 0 { // ... we didn't actually start yet... - checkpoint = math.MaxInt64 + checkpoint = math.MaxInt64 // start from the largest (possible) ID } else { // ... or we've already finished. - checkpoint = 0 + checkpoint = 0 // make following query no-op } } @@ -170,6 +179,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { // 2. If the history gets cleaned up between two migration trials, // the difference either just doesn't appear in the cache or - if already there - will be ignored later. + // Stream source data... sliceIdoHistory( ht.snapshot, "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ @@ -177,7 +187,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" < ? ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT ?", - nil, checkpoint, + nil, checkpoint, // ... since we were interrupted: func(idoRows []struct { Id uint64 ObjectId uint64 @@ -187,31 +197,40 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { var nhs []struct{ NextHardState uint8 } cacheSelect(*tx, &nhs, "SELECT next_hard_state FROM next_hard_state WHERE object_id=?", idoRow.ObjectId) - if len(nhs) < 1 { + if len(nhs) < 1 { // we just started (per checkable) + // At the moment (we're "travelling back in time") that's the checkable's hard state: cacheExec( *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", idoRow.ObjectId, idoRow.LastHardState, ) + // But for the current time point the previous hard state isn't known, yet: cacheExec( *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", idoRow.Id, idoRow.ObjectId, ) } else if idoRow.LastHardState == nhs[0].NextHardState { + // The hard state didn't change yet (per checkable), + // so this time point also awaits the previous hard state. cacheExec( *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", idoRow.Id, idoRow.ObjectId, ) - } else { + } else { // the hard state changed (per checkable) + // That past hard state is now available for the processed future time points: cacheExec( *tx, false, "INSERT INTO previous_hard_state(history_id, previous_hard_state) "+ "SELECT history_id, ? FROM next_ids WHERE object_id=?", idoRow.LastHardState, idoRow.ObjectId, ) + // Now they have what they wanted: onDeleted(cacheExec(*tx, false, "DELETE FROM next_hard_state WHERE object_id=?", idoRow.ObjectId)) onDeleted(cacheExec(*tx, false, "DELETE FROM next_ids WHERE object_id=?", idoRow.ObjectId)) + // That's done. + // Now do the same thing as in the "we just started" case above, for the same reason: + cacheExec( *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", idoRow.ObjectId, idoRow.LastHardState, @@ -232,11 +251,13 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { }, ) + // No past hard state is available for the processed future time points, assuming pending: cacheExec( *tx, false, "INSERT INTO previous_hard_state(history_id, previous_hard_state) "+ "SELECT history_id, 99 FROM next_ids", ) + // Now they should have what they wanted: onDeleted(cacheExec(*tx, false, "DELETE FROM next_hard_state")) onDeleted(cacheExec(*tx, false, "DELETE FROM next_ids")) }) @@ -244,10 +265,14 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { ht.bar.SetTotal(ht.bar.Current(), true) } +// chunkCacheTx rationale: during do operate on cache via *tx. On every completed operation call onNewUncommittedDml() +// which periodically commits *tx and starts a new tx. (That's why tx is a **, not just a *.) On every DELETE +// call onDeleted() which will cause a VACUUM after do if any rows were affected to save some space. +// (On non-recoverable errors the whole program exits.) func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func())) { logger := log.With("backend", "cache") - var deleted int64 - var inTx int + var totalAffectedByDeletes int64 + var onNewUncommittedDmlCallsSinceLastTx int tx, err := cache.Beginx() if err != nil { @@ -256,16 +281,16 @@ func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onDeleted func(sql.Resul do( &tx, - func(result sql.Result) { + func(result sql.Result) { // onDeleted if affected, err := result.RowsAffected(); err == nil { - deleted += affected + totalAffectedByDeletes += affected } else { log.Errorf("%+v", errors.Wrap(err, "can't get affected rows")) } }, - func() { - inTx++ - if inTx == bulk { + func() { // onNewUncommittedDml + onNewUncommittedDmlCallsSinceLastTx++ + if onNewUncommittedDmlCallsSinceLastTx == bulk { if err := tx.Commit(); err != nil { logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } @@ -277,7 +302,7 @@ func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onDeleted func(sql.Resul logger.Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) } - inTx = 0 + onNewUncommittedDmlCallsSinceLastTx = 0 } }, ) @@ -286,11 +311,12 @@ func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onDeleted func(sql.Resul logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } - if deleted > 0 { + if totalAffectedByDeletes > 0 { cacheExec(cache, true, "VACUUM") } } +// cacheGet does cache.Get(dest, query, args...). (On non-recoverable errors the whole program exits.) func cacheGet(cache interface { Get(dest interface{}, query string, args ...interface{}) error }, dest interface{}, query string, args ...interface{}) { @@ -300,6 +326,7 @@ func cacheGet(cache interface { } } +// cacheSelect does cacheTx.Select(dest, query, args...). (On non-recoverable errors the whole program exits.) func cacheSelect(cacheTx *sqlx.Tx, dest interface{}, query string, args ...interface{}) { if err := cacheTx.Select(dest, query, args...); err != nil { log.With("backend", "cache", "query", query, "args", args). @@ -307,6 +334,7 @@ func cacheSelect(cacheTx *sqlx.Tx, dest interface{}, query string, args ...inter } } +// cacheExec does cache.Exec(dml, args...). On non-recoverable errors the whole program exits if !allowFailure. func cacheExec(cache sqlx.Execer, allowFailure bool, dml string, args ...interface{}) sql.Result { res, err := cache.Exec(dml, args...) if err != nil { From 662744ee48218588dda54698b57f430d9c44005a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Oct 2021 13:54:05 +0200 Subject: [PATCH 032/103] cmd/ido2icingadb: migrate(): UPDATE after REPLACE ... not to loose data. --- cmd/ido2icingadb/main.go | 60 ++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index f4164bbd..cbc390ee 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -431,36 +431,6 @@ func migrate(c *Config, idb *icingadb.DB) { log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) } - for _, table := range res[0].Interface().([][]interface{}) { - if len(table) < 1 { - continue - } - - tRow := reflect.TypeOf(table[0]) - - update, ok := icingaDbUpdates[tRow] - if !ok { - update, _ = idb.BuildUpdateStmt(table[0]) - icingaDbUpdates[tRow] = update - } - - stmt, err := tx.PrepareNamed(update) - if err != nil { - log.With("backend", "Icinga DB", "dml", update). - Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) - } - - // UPDATE à one. - for _, row := range table { - if _, err := stmt.Exec(row); err != nil { - log.With("backend", "Icinga DB", "dml", update, "args", row). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) - } - } - - _ = stmt.Close() - } - for _, table := range res[1].Interface().([][]interface{}) { if len(table) < 1 { continue @@ -492,6 +462,36 @@ func migrate(c *Config, idb *icingadb.DB) { } } + for _, table := range res[0].Interface().([][]interface{}) { + if len(table) < 1 { + continue + } + + tRow := reflect.TypeOf(table[0]) + + update, ok := icingaDbUpdates[tRow] + if !ok { + update, _ = idb.BuildUpdateStmt(table[0]) + icingaDbUpdates[tRow] = update + } + + stmt, err := tx.PrepareNamed(update) + if err != nil { + log.With("backend", "Icinga DB", "dml", update). + Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) + } + + // UPDATE à one. + for _, row := range table { + if _, err := stmt.Exec(row); err != nil { + log.With("backend", "Icinga DB", "dml", update, "args", row). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + + _ = stmt.Close() + } + if err := tx.Commit(); err != nil { log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } From 4fe3134bddfd1188c2f6d3bea15ff38c564f3a3d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Oct 2021 14:53:28 +0200 Subject: [PATCH 033/103] cmd/ido2icingadb: document misc.go --- cmd/ido2icingadb/main.go | 5 +++ cmd/ido2icingadb/misc.go | 82 ++++++++++++++++++++++++++++------------ 2 files changed, 62 insertions(+), 25 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index cbc390ee..29a5c395 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -246,6 +246,11 @@ func computeProgress(c *Config, idb *icingadb.DB) { // Stream IDO IDs, ... sliceIdoHistory(ht.snapshot, query, nil, 0, func(rows []ProgressRow) (checkpoint interface{}) { + type convertedId struct { + ido uint64 + idb []byte + } + ids := make([]interface{}, 0, len(rows)) converted := make([]convertedId, 0, len(rows)) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index c7d7c2a7..bfe66e74 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -21,21 +21,23 @@ import ( "time" ) +// ProgressRow contains all IDO info needed to assemble an Icinga DB ID. type ProgressRow struct { - Id uint64 + // Id is the IDO table ID. + Id uint64 + // Name is the name of the affected comment or downtime. Name string } -type convertedId struct { - ido uint64 - idb []byte -} - +// barIncrementer simplifies incrementing bar. type barIncrementer struct { - bar *mpb.Bar + // bar is the bar to increment. + bar *mpb.Bar + // start shall be the work start time. start time.Time } +// inc increments bi.bar by i. func (bi *barIncrementer) inc(i int) { prev := bi.start now := time.Now() @@ -47,6 +49,7 @@ func (bi *barIncrementer) inc(i int) { const bulk = 10000 +// log is the root logger. var log = func() *zap.SugaredLogger { logger, err := zap.NewDevelopmentConfig().Build() if err != nil { @@ -56,6 +59,7 @@ var log = func() *zap.SugaredLogger { return logger.Sugar() }() +// objectTypes maps IDO values to Icinga DB ones. var objectTypes = map[uint8]string{1: "host", 2: "service"} // mkDeterministicUuid returns a formally random UUID (v4) as follows: 11111122-3300-4455-4455-555555555555 @@ -66,6 +70,8 @@ var objectTypes = map[uint8]string{1: "host", 2: "service"} // 3: "h" (for "history") // 4: the new UUID's formal version (unused bits zeroed) // 5: the ID of the row the new UUID is for in the IDO (big endian) +// +// Rationale: be able to pre-calculate the IDs to figure out which have already been migrated. func mkDeterministicUuid(table byte, rowId uint64) icingadbTypes.UUID { uid := uuidTemplate uid[3] = table @@ -82,7 +88,7 @@ func mkDeterministicUuid(table byte, rowId uint64) icingadbTypes.UUID { return icingadbTypes.UUID{UUID: uid} } -// uuidTemplate is for mkDeterministicUuid. +// uuidTemplate is for mkDeterministicUuid() to save a few CPU cycles. var uuidTemplate = func() uuid.UUID { buf := &bytes.Buffer{} buf.Write(uuid.Nil[:]) @@ -96,10 +102,12 @@ var uuidTemplate = func() uuid.UUID { return uid }() -// randomUuid generates a new UUIDv4. +// randomUuid generates a new UUIDv4. Saves getrandom(2) syscalls via bufio.Reader-s. +// (On non-recoverable errors the whole program exits.) func randomUuid() icingadbTypes.UUID { var rander *bufio.Reader + // Get random available rander. massRanders.Lock() for r := range massRanders.pool { rander = r @@ -108,6 +116,7 @@ func randomUuid() icingadbTypes.UUID { } massRanders.Unlock() + // Fall back to new one. if rander == nil { rander = bufio.NewReader(rand.Reader) } @@ -117,6 +126,7 @@ func randomUuid() icingadbTypes.UUID { log.Fatalf("%+v", errors.Wrap(err, "can't generate random UUID")) } + // Make it available for the next call. massRanders.Lock() massRanders.pool[rander] = struct{}{} massRanders.Unlock() @@ -124,6 +134,7 @@ func randomUuid() icingadbTypes.UUID { return icingadbTypes.UUID{UUID: id} } +// massRanders are randomUuid()'s storage. var massRanders = struct { sync.Mutex pool map[*bufio.Reader]struct{} @@ -132,7 +143,7 @@ var massRanders = struct { map[*bufio.Reader]struct{}{}, } -// hashAny combines PackAny and SHA1 hashing. +// hashAny combines objectpacker.PackAny and SHA1 hashing. func hashAny(in interface{}) []byte { hash := sha1.New() if err := objectpacker.PackAny(in, hash); err != nil { @@ -174,6 +185,7 @@ func calcServiceId(env, name1, name2 string) []byte { // Rationale: split the likely large result set of a query by adding a WHERE condition and a LIMIT, // both with ? placeholders. Due to this function's internals they have to be the last two placeholders. // checkpoint is the initial value for the WHERE condition, onRows() returns follow-up ones. +// (On non-recoverable errors the whole program exits.) func sliceIdoHistory(snapshot *sqlx.Tx, query string, args []interface{}, checkpoint, onRows interface{}) { vOnRows := reflect.ValueOf(onRows) // TODO: make onRows generic[T] one nice day @@ -205,27 +217,46 @@ func sliceIdoHistory(snapshot *sqlx.Tx, query string, args []interface{}, checkp } } +// historyType specifies a history data type. type historyType struct { - name string - idoTable string - idoIdColumn string - idoColumns []string - idbTable string - idbIdColumn string - convertId func(row ProgressRow, env string) []byte - cacheSchema []string - cacheFiller func(*historyType) + // name is a human-readable, but machine-friendly common name. + name string + // idoTable specifies the source table. + idoTable string + // idoIdColumn specifies idoTable's primary key. + idoIdColumn string + // idoColumns specifies idoTable's columns in addition to idoIdColumn computeProgress() needs. + idoColumns []string + // idbTable specifies the destination table computeProgress() compares the source data with. + idbTable string + // idoIdColumn specifies idbTable's primary key. + idbIdColumn string + // convertId converts the IDO row and the Icinga 2 env name to a value suitable for idbIdColumn. + convertId func(row ProgressRow, env string) []byte + // cacheSchema specifies .sqlite3's structure. + cacheSchema []string + // cacheFiller fills cache from snapshot. + cacheFiller func(*historyType) + // cacheLimitQuery rationale: see migrate(). cacheLimitQuery string - migrationQuery string - convertRows interface{} + // migrationQuery SELECTs source data for actual migration. + migrationQuery string + // convertRows intention: see migrate(). + convertRows interface{} - cache *sqlx.DB + // cache represents .sqlite3. + cache *sqlx.DB + // snapshot represents the data source. snapshot *sqlx.Tx - total int64 - bar *mpb.Bar - lastId uint64 + // total summarizes the source data. + total int64 + // bar represents the current progress bar. + bar *mpb.Bar + // lastId is the last already migrated ID. + lastId uint64 } +// setupBar (re-)initializes ht.bar. func (ht *historyType) setupBar(progress *mpb.Progress) { ht.bar = progress.AddBar( ht.total, @@ -240,6 +271,7 @@ func (ht *historyType) setupBar(progress *mpb.Progress) { type historyTypes [6]historyType +// forEach performs f per *ht in parallel. func (ht *historyTypes) forEach(f func(*historyType)) { eg, _ := errgroup.WithContext(context.Background()) for i := range *ht { From 77fa207eb50319a5ab4be4baf644236a2fe487ae Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 11 Oct 2021 18:12:12 +0200 Subject: [PATCH 034/103] cmd/ido2icingadb: simplify unnecessarily complex datatype --- cmd/ido2icingadb/cache.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index b4e9bff9..abdd9792 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -152,15 +152,15 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { } cacheGet(*tx, &nextIds, "SELECT COUNT(*) cnt, MIN(history_id) min_id FROM next_ids") - var previousHardState struct{ Cnt int64 } - cacheGet(*tx, &previousHardState, "SELECT COUNT(*) cnt FROM previous_hard_state") + var previousHardStateCnt int64 + cacheGet(*tx, &previousHardStateCnt, "SELECT COUNT(*) FROM previous_hard_state") var checkpoint int64 if nextIds.MinId.Valid { // there are next_ids checkpoint = nextIds.MinId.Int64 // this kind of caches is filled descending } else { // there aren't any next_ids // next_ids contains the most recently processed IDs and is only empty if... - if previousHardState.Cnt == 0 { + if previousHardStateCnt == 0 { // ... we didn't actually start yet... checkpoint = math.MaxInt64 // start from the largest (possible) ID } else { @@ -169,7 +169,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { } } - ht.bar.SetCurrent(previousHardState.Cnt + nextIds.Cnt) + ht.bar.SetCurrent(previousHardStateCnt + nextIds.Cnt) inc := barIncrementer{ht.bar, time.Now()} // We continue where we finished before. As we build the cache in reverse chronological order: From 068052f75b5f3e75789415d7fe58f7ce1fc3c2d3 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 12 Oct 2021 11:35:23 +0200 Subject: [PATCH 035/103] cmd/ido2icingadb: migrate(): don't REPLACE, but upsert (one by one) --- cmd/ido2icingadb/main.go | 80 ++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 53 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 29a5c395..865e16de 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -436,65 +436,39 @@ func migrate(c *Config, idb *icingadb.DB) { log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) } - for _, table := range res[1].Interface().([][]interface{}) { - if len(table) < 1 { - continue - } - - tRow := reflect.TypeOf(table[0]) - - insert, ok := icingaDbInserts[tRow] - if !ok { - insert, _ = idb.BuildInsertStmt(table[0]) - insert = "REPLACE " + strings.TrimPrefix(insert, "INSERT ") - icingaDbInserts[tRow] = insert - } - - for len(table) > 0 { - // REPLACE à 1000. - slice := table - if len(slice) > 1000 { - slice = slice[:1000] - table = table[1000:] - } else { - table = nil + for _, operation := range [...]struct { + data reflect.Value + buildStmt func(subject interface{}) (stmt string, _ int) + stmtCache map[reflect.Type]string + }{{res[1], idb.BuildUpsertStmt, icingaDbInserts}, {res[0], idb.BuildUpdateStmt, icingaDbUpdates}} { + for _, table := range operation.data.Interface().([][]interface{}) { + if len(table) < 1 { + continue } - if _, err := tx.NamedExec(insert, slice); err != nil { - log.With("backend", "Icinga DB", "dml", insert, "args", slice). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + tRow := reflect.TypeOf(table[0]) + + query, ok := operation.stmtCache[tRow] + if !ok { + query, _ = operation.buildStmt(table[0]) + operation.stmtCache[tRow] = query } - } - } - for _, table := range res[0].Interface().([][]interface{}) { - if len(table) < 1 { - continue - } - - tRow := reflect.TypeOf(table[0]) - - update, ok := icingaDbUpdates[tRow] - if !ok { - update, _ = idb.BuildUpdateStmt(table[0]) - icingaDbUpdates[tRow] = update - } - - stmt, err := tx.PrepareNamed(update) - if err != nil { - log.With("backend", "Icinga DB", "dml", update). - Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) - } - - // UPDATE à one. - for _, row := range table { - if _, err := stmt.Exec(row); err != nil { - log.With("backend", "Icinga DB", "dml", update, "args", row). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + stmt, err := tx.PrepareNamed(query) + if err != nil { + log.With("backend", "Icinga DB", "dml", query). + Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) } - } - _ = stmt.Close() + for _, row := range table { + if _, err := stmt.Exec(row); err != nil { + log.With("backend", "Icinga DB", "dml", query, "args", row). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + + _ = stmt.Close() + } } if err := tx.Commit(); err != nil { From 6e7e217b3c0c07ac1e53daf35e5487ce71105b21 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 12 Oct 2021 12:18:59 +0200 Subject: [PATCH 036/103] cmd/ido2icingadb: sliceIdoHistory(): take named args --- cmd/ido2icingadb/cache.go | 4 ++-- cmd/ido2icingadb/convert.go | 28 ++++++++++++++-------------- cmd/ido2icingadb/main.go | 8 ++++---- cmd/ido2icingadb/misc.go | 28 +++++++++++++++++++++------- 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index abdd9792..d2671534 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -50,7 +50,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ - ht.idoIdColumn+" > ? ORDER BY xh."+ht.idoIdColumn+" LIMIT ?", + ht.idoIdColumn+" > :checkpoint ORDER BY xh."+ht.idoIdColumn+" LIMIT :bulk", nil, checkpoint.MaxId.Int64, // ... since we were interrupted: func(idoRows []struct { Id uint64 @@ -186,7 +186,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ - ht.idoIdColumn+" < ? ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT ?", + ht.idoIdColumn+" < :checkpoint ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT :bulk", nil, checkpoint, // ... since we were interrupted: func(idoRows []struct { Id uint64 diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index e7642060..da44612a 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -19,9 +19,9 @@ const acknowledgementMigrationQuery = "SELECT ah.acknowledgement_id, UNIX_TIMEST "IFNULL(o.name2, '') name2 " + "FROM icinga_acknowledgements ah USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=ah.object_id " + - "WHERE ah.acknowledgement_id > ? " + // where we were interrupted + "WHERE ah.acknowledgement_id > :checkpoint " + // where we were interrupted "ORDER BY ah.acknowledgement_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT ?" + "LIMIT :bulk" // AckClear updates an already migrated ack event with the clear event info. type AckClear struct { @@ -183,9 +183,9 @@ const commentMigrationQuery = "SELECT ch.commenthistory_id, UNIX_TIMESTAMP(ch.en "o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + "FROM icinga_commenthistory ch USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=ch.object_id " + - "WHERE ch.commenthistory_id > ? " + // where we were interrupted + "WHERE ch.commenthistory_id > :checkpoint " + // where we were interrupted "ORDER BY ch.commenthistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT ?" + "LIMIT :bulk" type commentRow = struct { CommenthistoryId uint64 @@ -296,9 +296,9 @@ const downtimeMigrationQuery = "SELECT dh.downtimehistory_id, UNIX_TIMESTAMP(dh. "FROM icinga_downtimehistory dh USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=dh.object_id " + "LEFT JOIN icinga_scheduleddowntime sd ON sd.scheduleddowntime_id=dh.triggered_by_id " + - "WHERE dh.downtimehistory_id > ? " + // where we were interrupted + "WHERE dh.downtimehistory_id > :checkpoint " + // where we were interrupted "ORDER BY dh.downtimehistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT ?" + "LIMIT :bulk" type downtimeRow = struct { DowntimehistoryId uint64 @@ -440,9 +440,9 @@ const flappingMigrationQuery = "SELECT fh.flappinghistory_id, UNIX_TIMESTAMP(fh. "fh.high_threshold, o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + "FROM icinga_flappinghistory fh USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=fh.object_id " + - "WHERE fh.flappinghistory_id > ? " + // where we were interrupted + "WHERE fh.flappinghistory_id > :checkpoint " + // where we were interrupted "ORDER BY fh.flappinghistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT ?" + "LIMIT :bulk" // FlappingEnd updates an already migrated start event with the end event info. type FlappingEnd struct { @@ -598,10 +598,10 @@ const notificationMigrationQuery = "SELECT n.notification_id, n.notification_rea "n.long_output, n.contacts_notified, o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + "FROM icinga_notifications n USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=n.object_id " + - "WHERE n.notification_id <= ? AND " + - "n.notification_id > ? " + // where we were interrupted + "WHERE n.notification_id <= :cache_limit AND " + + "n.notification_id > :checkpoint " + // where we were interrupted "ORDER BY n.notification_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT ?" + "LIMIT :bulk" // zeroHash is a NULL alternative for NOT NULL columns. var zeroHash = make(icingadbTypes.Binary, 20) @@ -759,10 +759,10 @@ const stateMigrationQuery = "SELECT sh.statehistory_id, UNIX_TIMESTAMP(sh.state_ "sh.check_source, o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + "FROM icinga_statehistory sh USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=sh.object_id " + - "WHERE sh.statehistory_id <= ? AND " + - "sh.statehistory_id > ? " + // where we were interrupted + "WHERE sh.statehistory_id <= :cache_limit AND " + + "sh.statehistory_id > :checkpoint " + // where we were interrupted "ORDER BY sh.statehistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT ?" + "LIMIT :bulk" type stateRow = struct { StatehistoryId uint64 diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 865e16de..4d025eea 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -225,8 +225,8 @@ func computeProgress(c *Config, idb *icingadb.DB) { // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + - ht.idoIdColumn + " > ? ORDER BY xh." + // requires migrate() to migrate serially, in order - ht.idoIdColumn + " LIMIT ?" + ht.idoIdColumn + " > :checkpoint ORDER BY xh." + // requires migrate() to migrate serially, in order + ht.idoIdColumn + " LIMIT :bulk" // As long as the current chunk is lastRowsLen long (doesn't change)... var lastRowsLen int @@ -405,14 +405,14 @@ func migrate(c *Config, idb *icingadb.DB) { // and the rows (below) } - var args []interface{} + var args map[string]interface{} // For the case that the cache was older that the IDO, // but ht.cacheFiller couldn't update it, limit (WHERE) our source data set. if ht.cacheLimitQuery != "" { var limit uint64 cacheGet(ht.cache, &limit, ht.cacheLimitQuery) - args = append(args, limit) + args = map[string]interface{}{"cache_limit": limit} } icingaDbInserts := map[reflect.Type]string{} diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index bfe66e74..66b7e26b 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -180,13 +180,13 @@ func calcServiceId(env, name1, name2 string) []byte { return hashAny([2]string{env, name1 + "!" + name2}) } -// sliceIdoHistory performs query with args+[]interface{}{checkpoint,bulk} on snapshot and passes the results -// to onRows (a func([]T)interface{}) until either an empty result set or onRows() returns nil. +// sliceIdoHistory performs query with args+map[string]interface{}{"checkpoint": checkpoint, "bulk": bulk} on snapshot +// and passes the results to onRows (a func([]T)interface{}) until either an empty result set or onRows() returns nil. // Rationale: split the likely large result set of a query by adding a WHERE condition and a LIMIT, -// both with ? placeholders. Due to this function's internals they have to be the last two placeholders. +// both with :named placeholders (:checkpoint, :bulk). // checkpoint is the initial value for the WHERE condition, onRows() returns follow-up ones. // (On non-recoverable errors the whole program exits.) -func sliceIdoHistory(snapshot *sqlx.Tx, query string, args []interface{}, checkpoint, onRows interface{}) { +func sliceIdoHistory(snapshot *sqlx.Tx, query string, args map[string]interface{}, checkpoint, onRows interface{}) { vOnRows := reflect.ValueOf(onRows) // TODO: make onRows generic[T] one nice day tRows := vOnRows.Type(). // func(rows []T) (checkpoint interface{}) @@ -197,13 +197,27 @@ func sliceIdoHistory(snapshot *sqlx.Tx, query string, args []interface{}, checkp vRows := vNewRows.Elem() onRowsArgs := [1]reflect.Value{vRows} vZeroRows := reflect.Zero(tRows) - args = append(append([]interface{}(nil), args...), checkpoint, bulk) + + if args == nil { + args = map[string]interface{}{} + } + + args["checkpoint"] = checkpoint + args["bulk"] = bulk for { - if err := snapshot.Select(rowsPtr, query, args...); err != nil { + // TODO: use Tx#SelectNamed() one nice day (https://github.com/jmoiron/sqlx/issues/779) + stmt, err := snapshot.PrepareNamed(query) + if err != nil { + log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + + if err := stmt.Select(rowsPtr, args); err != nil { log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) } + _ = stmt.Close() + if vRows.Len() < 1 { break } @@ -213,7 +227,7 @@ func sliceIdoHistory(snapshot *sqlx.Tx, query string, args []interface{}, checkp } vRows.Set(vZeroRows) - args[len(args)-2] = checkpoint + args["checkpoint"] = checkpoint } } From bc5ad21a6d71704f780176f37835a216b1645e94 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 2 Dec 2021 12:44:50 +0100 Subject: [PATCH 037/103] cmd/ido2icingadb: simplify computeProgress() a lot --- cmd/ido2icingadb/main.go | 149 ++++++++++----------------------------- cmd/ido2icingadb/misc.go | 36 ---------- 2 files changed, 36 insertions(+), 149 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 4d025eea..d72a1358 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -19,7 +19,6 @@ import ( "os" "path" "reflect" - "strings" "time" ) @@ -72,16 +71,13 @@ func main() { log.Info("Computing progress") - // Count total source data, so the following computeProgress() - // knows how many work is left and can display progress bars. + // Count total source data, so the following functions + // know how many work is left and can display progress bars. countIdoHistory() - // Roll out a red carpet for the following computeProgress()' progress bars. - _ = log.Sync() - // computeProgress figures out which data has already been migrated // not to start from the beginning every time in the following migrate(). - computeProgress(c, idb) + computeProgress(idb) // On rationale read buildEventTimeCache() and buildPreviousHardStateCache() docs. log.Info("Filling cache") @@ -208,119 +204,35 @@ func countIdoHistory() { // computeProgress initializes types[*].lastId with how many events have already been migrated to idb. // (On non-recoverable errors the whole program exits.) -func computeProgress(c *Config, idb *icingadb.DB) { - progress := mpb.New() - for i := range types { - types[i].setupBar(progress) +func computeProgress(idb *icingadb.DB) { + { + _, err := idb.Exec(`CREATE TABLE IF NOT EXISTS ido_migration_progress ( + history_type VARCHAR(63) PRIMARY KEY, + last_ido_id BIGINT unsigned NOT NULL +)`) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't create table ido_migration_progress")) + } } types.forEach(func(ht *historyType) { - if ht.total == 0 { - ht.bar.SetTotal(ht.bar.Current(), true) - return + stmt := idb.Rebind( + "INSERT INTO ido_migration_progress(history_type, last_ido_id) " + + "VALUES (?, 0) ON DUPLICATE KEY UPDATE history_type=history_type", + ) + + if _, err := idb.Exec(stmt, ht.name); err != nil { + log.With("backend", "Icinga DB", "dml", stmt, "args", []interface{}{ht.name}). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) } - query := "SELECT xh." + - strings.Join(append(append([]string(nil), ht.idoColumns...), ht.idoIdColumn), ", xh.") + " id FROM " + - // For actual migration icinga_objects will be joined anyway, - // so it makes no sense to take vanished objects into account. - ht.idoTable + " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE " + - ht.idoIdColumn + " > :checkpoint ORDER BY xh." + // requires migrate() to migrate serially, in order - ht.idoIdColumn + " LIMIT :bulk" + const query = "SELECT last_ido_id FROM ido_migration_progress WHERE history_type=?" - // As long as the current chunk is lastRowsLen long (doesn't change)... - var lastRowsLen int - - // ... we can re-use these: - var lastQuery string - var lastStmt *sqlx.Stmt - - baseIdbQuery := fmt.Sprintf("SELECT %s FROM %s WHERE %s IN (?)", ht.idbIdColumn, ht.idbTable, ht.idbIdColumn) - inc := barIncrementer{ht.bar, time.Now()} - - defer func() { - if lastStmt != nil { - _ = lastStmt.Close() - } - }() - - // Stream IDO IDs, ... - sliceIdoHistory(ht.snapshot, query, nil, 0, func(rows []ProgressRow) (checkpoint interface{}) { - type convertedId struct { - ido uint64 - idb []byte - } - - ids := make([]interface{}, 0, len(rows)) - converted := make([]convertedId, 0, len(rows)) - - // ... convert them to Icinga DB ones, ... - for _, row := range rows { - conv := ht.convertId(row, c.Icinga2.Env) - ids = append(ids, conv) - converted = append(converted, convertedId{row.Id, conv}) - } - - // ... prepare a new query if the last one doesn't fit, ... - if len(rows) != lastRowsLen { - if lastStmt != nil { - _ = lastStmt.Close() - } - - lastRowsLen = len(rows) - - { - var err error - lastQuery, _, err = sqlx.In(baseIdbQuery, ids...) - - if err != nil { - log.With("query", baseIdbQuery).Fatalf("%+v", errors.Wrap(err, "can't assemble query")) - } - } - - var err error - lastStmt, err = idb.Preparex(lastQuery) - - if err != nil { - log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - } - - // ... select which have already been migrated, ... - var present [][]byte - if err := lastStmt.Select(&present, ids...); err != nil { - log.With("query", lastQuery).Fatalf("%+v", errors.Wrap(err, "can't perform query")) - } - - presentSet := map[[20]byte]struct{}{} - for _, row := range present { - var key [20]byte - copy(key[:], row) - presentSet[key] = struct{}{} - } - - // ... and in IDO ID order: - for _, conv := range converted { - var key [20]byte - copy(key[:], conv.idb) - - // Stop on the first not yet migrated. - if _, ok := presentSet[key]; !ok { - return nil - } - - // If an ID has already been migrated, increase the actual migration's start point. - ht.lastId = conv.ido - } - - inc.inc(len(rows)) - return ht.lastId - }) - - ht.bar.SetTotal(ht.bar.Current(), true) + if err := idb.Get(&ht.lastId, query, ht.name); err != nil { + log.With("backend", "Icinga DB", "query", query, "args", []interface{}{ht.name}). + Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } }) - - progress.Wait() } // fillCache fills /.sqlite3 (actually types[*].cacheFiller does). @@ -471,6 +383,17 @@ func migrate(c *Config, idb *icingadb.DB) { } } + if lastIdoId := res[2].Interface(); lastIdoId != nil { + const stmt = "UPDATE ido_migration_progress SET last_ido_id=:last_ido_id " + + "WHERE history_type=:history_type" + + args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} + if _, err := tx.NamedExec(stmt, args); err != nil { + log.With("backend", "Icinga DB", "dml", stmt, "args", args). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + if err := tx.Commit(); err != nil { log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 66b7e26b..3cbc6046 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -21,14 +21,6 @@ import ( "time" ) -// ProgressRow contains all IDO info needed to assemble an Icinga DB ID. -type ProgressRow struct { - // Id is the IDO table ID. - Id uint64 - // Name is the name of the affected comment or downtime. - Name string -} - // barIncrementer simplifies incrementing bar. type barIncrementer struct { // bar is the bar to increment. @@ -239,14 +231,6 @@ type historyType struct { idoTable string // idoIdColumn specifies idoTable's primary key. idoIdColumn string - // idoColumns specifies idoTable's columns in addition to idoIdColumn computeProgress() needs. - idoColumns []string - // idbTable specifies the destination table computeProgress() compares the source data with. - idbTable string - // idoIdColumn specifies idbTable's primary key. - idbIdColumn string - // convertId converts the IDO row and the Icinga 2 env name to a value suitable for idbIdColumn. - convertId func(row ProgressRow, env string) []byte // cacheSchema specifies .sqlite3's structure. cacheSchema []string // cacheFiller fills cache from snapshot. @@ -304,9 +288,6 @@ var types = historyTypes{ name: "acknowledgement", idoTable: "icinga_acknowledgements", idoIdColumn: "acknowledgement_id", - idbTable: "history", - idbIdColumn: "id", - convertId: func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('a', row.Id); return u.UUID[:] }, cacheSchema: eventTimeCacheSchema, cacheFiller: func(ht *historyType) { buildEventTimeCache(ht, []string{ @@ -321,10 +302,6 @@ var types = historyTypes{ name: "comment", idoTable: "icinga_commenthistory", idoIdColumn: "commenthistory_id", - idoColumns: []string{"name"}, - idbTable: "comment_history", - idbIdColumn: "comment_id", - convertId: func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, migrationQuery: commentMigrationQuery, convertRows: convertCommentRows, }, @@ -332,10 +309,6 @@ var types = historyTypes{ name: "downtime", idoTable: "icinga_downtimehistory", idoIdColumn: "downtimehistory_id", - idoColumns: []string{"name"}, - idbTable: "downtime_history", - idbIdColumn: "downtime_id", - convertId: func(row ProgressRow, env string) []byte { return calcObjectId(env, row.Name) }, migrationQuery: downtimeMigrationQuery, convertRows: convertDowntimeRows, }, @@ -343,9 +316,6 @@ var types = historyTypes{ name: "flapping", idoTable: "icinga_flappinghistory", idoIdColumn: "flappinghistory_id", - idbTable: "history", - idbIdColumn: "id", - convertId: func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('f', row.Id); return u.UUID[:] }, cacheSchema: eventTimeCacheSchema, cacheFiller: func(ht *historyType) { buildEventTimeCache(ht, []string{ @@ -360,9 +330,6 @@ var types = historyTypes{ name: "notification", idoTable: "icinga_notifications", idoIdColumn: "notification_id", - idbTable: "notification_history", - idbIdColumn: "id", - convertId: func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('n', row.Id); return u.UUID[:] }, cacheSchema: previousHardStateCacheSchema, cacheFiller: func(ht *historyType) { buildPreviousHardStateCache(ht, []string{ @@ -377,9 +344,6 @@ var types = historyTypes{ name: "state", idoTable: "icinga_statehistory", idoIdColumn: "statehistory_id", - idbTable: "state_history", - idbIdColumn: "id", - convertId: func(row ProgressRow, _ string) []byte { u := mkDeterministicUuid('s', row.Id); return u.UUID[:] }, cacheSchema: previousHardStateCacheSchema, cacheFiller: func(ht *historyType) { buildPreviousHardStateCache(ht, []string{"xh.statehistory_id id", "xh.object_id", "xh.last_hard_state"}) From 2c23b145b72275625c1a7048e717af5b0fba4a63 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 2 Dec 2021 13:17:03 +0100 Subject: [PATCH 038/103] cmd/ido2icingadb: resume migration progress bar where interrupted --- cmd/ido2icingadb/main.go | 21 +++++++++++++++++++-- cmd/ido2icingadb/misc.go | 2 ++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index d72a1358..e5ba59ac 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -202,8 +202,7 @@ func countIdoHistory() { }) } -// computeProgress initializes types[*].lastId with how many events have already been migrated to idb. -// (On non-recoverable errors the whole program exits.) +// computeProgress initializes types[*].lastId and types[*].done. (On non-recoverable errors the whole program exits.) func computeProgress(idb *icingadb.DB) { { _, err := idb.Exec(`CREATE TABLE IF NOT EXISTS ido_migration_progress ( @@ -233,6 +232,22 @@ func computeProgress(idb *icingadb.DB) { Fatalf("%+v", errors.Wrap(err, "can't perform query")) } }) + + types.forEach(func(ht *historyType) { + err := ht.snapshot.Get( + &ht.done, + // For actual migration icinga_objects will be joined anyway, + // so it makes no sense to take vanished objects into account. + "SELECT COUNT(*) FROM "+ht.idoTable+" xh USE INDEX (PRIMARY)"+ + " INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ht.idoIdColumn+"<=?", + ht.lastId, + ) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't count query")) + } + + log.Infow("Counted migrated IDO events", "type", ht.name, "amount", ht.done) + }) } // fillCache fills /.sqlite3 (actually types[*].cacheFiller does). @@ -329,6 +344,8 @@ func migrate(c *Config, idb *icingadb.DB) { icingaDbInserts := map[reflect.Type]string{} icingaDbUpdates := map[reflect.Type]string{} + + ht.bar.SetCurrent(ht.done) inc := barIncrementer{ht.bar, time.Now()} // Stream IDO rows, ... diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 3cbc6046..79d57efd 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -248,6 +248,8 @@ type historyType struct { snapshot *sqlx.Tx // total summarizes the source data. total int64 + // done summarizes the migrated data. + done int64 // bar represents the current progress bar. bar *mpb.Bar // lastId is the last already migrated ID. From b78b15e32c4ed4f86dd226a769577e7c7b29fab5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 2 Dec 2021 13:18:19 +0100 Subject: [PATCH 039/103] cmd/ido2icingadb: ensure migration progress bar finishes --- cmd/ido2icingadb/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index e5ba59ac..c2a9d602 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -420,6 +420,8 @@ func migrate(c *Config, idb *icingadb.DB) { }, ).Interface(), ) + + ht.bar.SetTotal(ht.bar.Current(), true) }) progress.Wait() From 34b3ee1767d5a91f5f00cf2486690df493af6240 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 2 Dec 2021 14:12:24 +0100 Subject: [PATCH 040/103] cmd/ido2icingadb: merge countIdoHistory() and computeProgress() --- cmd/ido2icingadb/main.go | 48 +++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index c2a9d602..cca3d335 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -71,10 +71,6 @@ func main() { log.Info("Computing progress") - // Count total source data, so the following functions - // know how many work is left and can display progress bars. - countIdoHistory() - // computeProgress figures out which data has already been migrated // not to start from the beginning every time in the following migrate(). computeProgress(idb) @@ -184,25 +180,8 @@ func startIdoTx(ido *icingadb.DB) { }) } -// countIdoHistory initializes types[*].total with how many events to migrate. +// computeProgress initializes types[*].lastId, types[*].total and types[*].done. // (On non-recoverable errors the whole program exits.) -func countIdoHistory() { - types.forEach(func(ht *historyType) { - err := ht.snapshot.Get( - &ht.total, - // For actual migration icinga_objects will be joined anyway, - // so it makes no sense to take vanished objects into account. - "SELECT COUNT(*) FROM "+ht.idoTable+" xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id", - ) - if err != nil { - log.Fatalf("%+v", errors.Wrap(err, "can't count query")) - } - - log.Infow("Counted total IDO events", "type", ht.name, "amount", ht.total) - }) -} - -// computeProgress initializes types[*].lastId and types[*].done. (On non-recoverable errors the whole program exits.) func computeProgress(idb *icingadb.DB) { { _, err := idb.Exec(`CREATE TABLE IF NOT EXISTS ido_migration_progress ( @@ -234,19 +213,32 @@ func computeProgress(idb *icingadb.DB) { }) types.forEach(func(ht *historyType) { - err := ht.snapshot.Get( - &ht.done, + var rows []struct { + Migrated uint8 + Cnt int64 + } + + err := ht.snapshot.Select( + &rows, // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. - "SELECT COUNT(*) FROM "+ht.idoTable+" xh USE INDEX (PRIMARY)"+ - " INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ht.idoIdColumn+"<=?", - ht.lastId, + "SELECT xh."+ht.idoIdColumn+"<=? migrated, COUNT(*) cnt FROM "+ht.idoTable+" xh"+ + " INNER JOIN icinga_objects o ON o.object_id=xh.object_id GROUP BY xh."+ht.idoIdColumn+"<=?", + ht.lastId, ht.lastId, ) if err != nil { log.Fatalf("%+v", errors.Wrap(err, "can't count query")) } - log.Infow("Counted migrated IDO events", "type", ht.name, "amount", ht.done) + for _, row := range rows { + ht.total += row.Cnt + + if row.Migrated == 1 { + ht.done = row.Cnt + } + } + + log.Infow("Counted migrated IDO events", "type", ht.name, "migrated", ht.done, "total", ht.total) }) } From 4e58631f8e1a5ff6fc06d053a0328300d8e72a79 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 2 Dec 2021 17:23:51 +0100 Subject: [PATCH 041/103] cmd/ido2icingadb: make IDs UUID -> SHA1 --- cmd/ido2icingadb/convert.go | 92 ++++++++++++++++++++++------------- cmd/ido2icingadb/misc.go | 87 --------------------------------- cmd/ido2icingadb/misc_test.go | 13 ----- 3 files changed, 58 insertions(+), 134 deletions(-) delete mode 100644 cmd/ido2icingadb/misc_test.go diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index da44612a..e2e0e3a7 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -9,6 +9,7 @@ import ( "github.com/icinga/icingadb/pkg/utils" "github.com/jmoiron/sqlx" "github.com/pkg/errors" + "strconv" "strings" "time" ) @@ -98,14 +99,11 @@ func convertAcknowledgementRows( name += "!" + row.Name2 } - id := mkDeterministicUuid('a', row.AcknowledgementId) typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) serviceId := calcServiceId(env, row.Name1, row.Name2) - - acknowledgementHistoryId := hashAny([]interface{}{ - env, strings.Title(typ), name, float64(utils.UnixMilli(set.Time())), - }) + setTime := float64(utils.UnixMilli(set.Time())) + acknowledgementHistoryId := hashAny([]interface{}{env, name, setTime}) if row.AcknowledgementType == 0 { // clear // The set counterpart should already have been inserted. @@ -115,7 +113,9 @@ func convertAcknowledgementRows( h := &history.HistoryAck{ HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: id}, + HistoryEntity: history.HistoryEntity{ + Id: hashAny([]interface{}{env, "ack_clear", name, setTime}), + }, EnvironmentId: envId, EndpointId: endpointId, ObjectType: typ, @@ -152,7 +152,9 @@ func convertAcknowledgementRows( h := &history.HistoryAck{ HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: id}, + HistoryEntity: history.HistoryEntity{ + Id: hashAny([]interface{}{env, "ack_set", name, setTime}), + }, EnvironmentId: envId, EndpointId: endpointId, ObjectType: typ, @@ -242,7 +244,7 @@ func convertCommentRows( h1 := &history.HistoryComment{ HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: randomUuid()}, + HistoryEntity: history.HistoryEntity{Id: hashAny([]string{env, "comment_add", row.Name})}, EnvironmentId: envId, EndpointId: endpointId, ObjectType: typ, @@ -260,7 +262,7 @@ func convertCommentRows( if !removeTime.Time().IsZero() { // remove h2 := &history.HistoryComment{ HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: randomUuid()}, + HistoryEntity: history.HistoryEntity{Id: hashAny([]string{env, "comment_remove", row.Name})}, EnvironmentId: envId, EndpointId: endpointId, ObjectType: typ, @@ -391,7 +393,7 @@ func convertDowntimeRows( h1 := &history.HistoryDowntime{ HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: randomUuid()}, + HistoryEntity: history.HistoryEntity{Id: hashAny([]string{env, "downtime_start", row.Name})}, EnvironmentId: envId, EndpointId: endpointId, ObjectType: typ, @@ -409,7 +411,7 @@ func convertDowntimeRows( if !actualEnd.Time().IsZero() { // remove h2 := &history.HistoryDowntime{ HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: randomUuid()}, + HistoryEntity: history.HistoryEntity{Id: hashAny([]string{env, "downtime_end", row.Name})}, EnvironmentId: envId, EndpointId: endpointId, ObjectType: typ, @@ -517,14 +519,11 @@ func convertFlappingRows( name += "!" + row.Name2 } - id := mkDeterministicUuid('f', row.FlappinghistoryId) typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) serviceId := calcServiceId(env, row.Name1, row.Name2) - - flappingHistoryId := hashAny([]interface{}{ - env, strings.Title(typ), name, float64(utils.UnixMilli(start.Time())), - }) + startTime := float64(utils.UnixMilli(start.Time())) + flappingHistoryId := hashAny([]interface{}{env, name, startTime}) if row.EventType == 1001 { // end // The start counterpart should already have been inserted. @@ -532,7 +531,9 @@ func convertFlappingRows( h := &history.HistoryFlapping{ HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: id}, + HistoryEntity: history.HistoryEntity{ + Id: hashAny([]interface{}{env, "flapping_end", name, startTime}), + }, EnvironmentId: envId, EndpointId: endpointId, ObjectType: typ, @@ -569,7 +570,9 @@ func convertFlappingRows( h := &history.HistoryFlapping{ HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: id}, + HistoryEntity: history.HistoryEntity{ + Id: hashAny([]interface{}{env, "flapping_start", name, startTime}), + }, EnvironmentId: envId, EndpointId: endpointId, ObjectType: typ, @@ -603,9 +606,6 @@ const notificationMigrationQuery = "SELECT n.notification_id, n.notification_rea "ORDER BY n.notification_id " + // allows computeProgress() not to check all IDO rows for whether migrated "LIMIT :bulk" -// zeroHash is a NULL alternative for NOT NULL columns. -var zeroHash = make(icingadbTypes.Binary, 20) - // notificationTypes maps IDO values to Icinga DB ones. var notificationTypes = map[uint8]icingadbTypes.NotificationType{5: 1, 6: 2, 7: 4, 8: 8, 1: 16, 2: 128, 3: 256} @@ -678,11 +678,10 @@ func convertNotificationRows( continue } - id := mkDeterministicUuid('n', row.NotificationId) - typ := objectTypes[row.ObjecttypeId] - hostId := calcObjectId(env, row.Name1) - serviceId := calcServiceId(env, row.Name1, row.Name2) - ts := convertTime(row.EndTime, row.EndTimeUsec) + // The IDO tracks only sent notifications, but not notification config objects. We have to improvise. + name := strings.Join( + []string{row.Name1, row.Name2, "migrated from IDO", strconv.FormatUint(row.NotificationId, 36)}, "!", + ) var nt icingadbTypes.NotificationType if row.NotificationReason == 0 { @@ -695,13 +694,30 @@ func convertNotificationRows( nt = notificationTypes[row.NotificationReason] } + ntEnum, err := nt.Value() + if err != nil { + // Programming error + panic(err) + } + + ts := convertTime(row.EndTime, row.EndTimeUsec) + notificationHistoryId := hashAny([]interface{}{env, name, ntEnum, ts}) + id := hashAny([]interface{}{env, "notification", name, ntEnum, ts}) + typ := objectTypes[row.ObjecttypeId] + hostId := calcObjectId(env, row.Name1) + serviceId := calcServiceId(env, row.Name1, row.Name2) + text := row.Output if row.LongOutput.Valid { text += "\n\n" + row.LongOutput.String } notificationHistory = append(notificationHistory, &history.NotificationHistory{ - HistoryTableEntity: history.HistoryTableEntity{Id: id}, + HistoryTableEntity: history.HistoryTableEntity{ + EntityWithoutChecksum: v1.EntityWithoutChecksum{ + IdMeta: v1.IdMeta{Id: notificationHistoryId}, + }, + }, HistoryTableMeta: history.HistoryTableMeta{ EnvironmentId: envId, EndpointId: endpointId, @@ -709,7 +725,7 @@ func convertNotificationRows( HostId: hostId, ServiceId: serviceId, }, - NotificationId: zeroHash, + NotificationId: calcObjectId(env, name), Type: nt, SendTime: ts, State: row.State, @@ -729,7 +745,7 @@ func convertNotificationRows( ServiceId: serviceId, EventType: "notification", }, - NotificationHistoryId: id, + NotificationHistoryId: notificationHistoryId, EventTime: ts, }) @@ -738,7 +754,9 @@ func convertNotificationRows( userNotificationHistory = append(userNotificationHistory, &history.UserNotificationHistory{ EntityWithoutChecksum: v1.EntityWithoutChecksum{ - IdMeta: v1.IdMeta{Id: utils.Checksum(append(append([]byte(nil), id.UUID[:]...), userId...))}, + IdMeta: v1.IdMeta{ + Id: utils.Checksum(append(append([]byte(nil), notificationHistoryId...), userId...)), + }, }, EnvironmentMeta: v1.EnvironmentMeta{EnvironmentId: envId}, NotificationHistoryId: id, @@ -811,14 +829,20 @@ func convertStateRows( continue } - id := mkDeterministicUuid('s', row.StatehistoryId) + name := strings.Join([]string{row.Name1, row.Name2}, "!") + ts := convertTime(row.StateTime, row.StateTimeUsec) + stateHistoryId := hashAny([]interface{}{env, name, ts}) + id := hashAny([]interface{}{env, "state_change", name, ts}) typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) serviceId := calcServiceId(env, row.Name1, row.Name2) - ts := convertTime(row.StateTime, row.StateTimeUsec) stateHistory = append(stateHistory, &history.StateHistory{ - HistoryTableEntity: history.HistoryTableEntity{Id: id}, + HistoryTableEntity: history.HistoryTableEntity{ + EntityWithoutChecksum: v1.EntityWithoutChecksum{ + IdMeta: v1.IdMeta{Id: stateHistoryId}, + }, + }, HistoryTableMeta: history.HistoryTableMeta{ EnvironmentId: envId, EndpointId: endpointId, @@ -849,7 +873,7 @@ func convertStateRows( ServiceId: serviceId, EventType: "state_change", }, - StateHistoryId: id, + StateHistoryId: stateHistoryId, EventTime: ts, }) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 79d57efd..1590eaf9 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -1,13 +1,8 @@ package main import ( - "bufio" - "bytes" "context" - "crypto/rand" "crypto/sha1" - "encoding/binary" - "github.com/google/uuid" "github.com/icinga/icingadb/pkg/icingadb/objectpacker" icingadbTypes "github.com/icinga/icingadb/pkg/types" "github.com/jmoiron/sqlx" @@ -17,7 +12,6 @@ import ( "go.uber.org/zap" "golang.org/x/sync/errgroup" "reflect" - "sync" "time" ) @@ -54,87 +48,6 @@ var log = func() *zap.SugaredLogger { // objectTypes maps IDO values to Icinga DB ones. var objectTypes = map[uint8]string{1: "host", 2: "service"} -// mkDeterministicUuid returns a formally random UUID (v4) as follows: 11111122-3300-4455-4455-555555555555 -// -// 0: zeroed -// 1: "IDO" (where the data identified by the new UUID is from) -// 2: the history table the new UUID is for, e.g. "s" for state_history -// 3: "h" (for "history") -// 4: the new UUID's formal version (unused bits zeroed) -// 5: the ID of the row the new UUID is for in the IDO (big endian) -// -// Rationale: be able to pre-calculate the IDs to figure out which have already been migrated. -func mkDeterministicUuid(table byte, rowId uint64) icingadbTypes.UUID { - uid := uuidTemplate - uid[3] = table - - buf := &bytes.Buffer{} - if err := binary.Write(buf, binary.BigEndian, rowId); err != nil { - panic(err) - } - - bEId := buf.Bytes() - uid[7] = bEId[0] - copy(uid[9:], bEId[1:]) - - return icingadbTypes.UUID{UUID: uid} -} - -// uuidTemplate is for mkDeterministicUuid() to save a few CPU cycles. -var uuidTemplate = func() uuid.UUID { - buf := &bytes.Buffer{} - buf.Write(uuid.Nil[:]) - - uid, err := uuid.NewRandomFromReader(buf) - if err != nil { - panic(err) - } - - copy(uid[:], "IDO h") - return uid -}() - -// randomUuid generates a new UUIDv4. Saves getrandom(2) syscalls via bufio.Reader-s. -// (On non-recoverable errors the whole program exits.) -func randomUuid() icingadbTypes.UUID { - var rander *bufio.Reader - - // Get random available rander. - massRanders.Lock() - for r := range massRanders.pool { - rander = r - delete(massRanders.pool, r) - break - } - massRanders.Unlock() - - // Fall back to new one. - if rander == nil { - rander = bufio.NewReader(rand.Reader) - } - - id, err := uuid.NewRandomFromReader(rander) - if err != nil { - log.Fatalf("%+v", errors.Wrap(err, "can't generate random UUID")) - } - - // Make it available for the next call. - massRanders.Lock() - massRanders.pool[rander] = struct{}{} - massRanders.Unlock() - - return icingadbTypes.UUID{UUID: id} -} - -// massRanders are randomUuid()'s storage. -var massRanders = struct { - sync.Mutex - pool map[*bufio.Reader]struct{} -}{ - sync.Mutex{}, - map[*bufio.Reader]struct{}{}, -} - // hashAny combines objectpacker.PackAny and SHA1 hashing. func hashAny(in interface{}) []byte { hash := sha1.New() diff --git a/cmd/ido2icingadb/misc_test.go b/cmd/ido2icingadb/misc_test.go deleted file mode 100644 index e3266e88..00000000 --- a/cmd/ido2icingadb/misc_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "bytes" - "testing" -) - -func TestMkDeterministicUuid(t *testing.T) { - id := mkDeterministicUuid('s', 0x0102030405060708).UUID - if !bytes.Equal(id[:], []byte{'I', 'D', 'O', 's', 'h', 0, 0x40, 1, 0x80, 2, 3, 4, 5, 6, 7, 8}) { - t.Error("got wrong UUID from mkDeterministicUuid(stateHistory, 0x0102030405060708)") - } -} From 163c8629e93dcb750d34aa359aaea124e3f24c37 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 2 Dec 2021 19:24:52 +0100 Subject: [PATCH 042/103] cmd/ido2icingadb: env: "Environment" config constant value -> hex ID --- cmd/ido2icingadb/main.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index cca3d335..a8f68e59 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha1" "database/sql" + "encoding/hex" "fmt" "github.com/goccy/go-yaml" "github.com/icinga/icingadb/pkg/config" @@ -36,7 +37,7 @@ type Config struct { IcingaDB config.Database `yaml:"icingadb"` // Icinga2 specifies information the IDO doesn't provide. Icinga2 struct { - // Env specifies the "Environment" config constant value (likely ""). + // Env specifies the environment ID, hex. Env string `yaml:"env"` // Endpoint specifies the name on the main endpoint writing to IDO. Endpoint string `yaml:"endpoint"` @@ -56,6 +57,12 @@ func main() { os.Exit(ex) } + envId, err := hex.DecodeString(c.Icinga2.Env) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "bad env ID: %s\n", err.Error()) + os.Exit(2) + } + defer func() { _ = log.Sync() }() log.Info("Starting IDO to Icinga DB history migration") @@ -80,7 +87,7 @@ func main() { fillCache() log.Info("Actually migrating") - migrate(c, idb) + migrate(c, idb, envId) } // parseConfig validates the f.Config file and returns the config and -1 or - on failure - nil and an exit code. @@ -265,8 +272,7 @@ var tAny = reflect.TypeOf((*interface{})(nil)). // *interface{} Elem() // interface{} // migrate does the actual migration. -func migrate(c *Config, idb *icingadb.DB) { - envId := sha1.Sum([]byte(c.Icinga2.Env)) +func migrate(c *Config, idb *icingadb.DB, envId []byte) { endpointId := sha1.Sum([]byte(c.Icinga2.Endpoint)) progress := mpb.New() @@ -295,7 +301,7 @@ func migrate(c *Config, idb *icingadb.DB) { }() vConvertRowsArgs := [6]reflect.Value{ - reflect.ValueOf(c.Icinga2.Env), reflect.ValueOf(icingadbTypes.Binary(envId[:])), + reflect.ValueOf(c.Icinga2.Env), reflect.ValueOf(icingadbTypes.Binary(envId)), reflect.ValueOf(icingadbTypes.Binary(endpointId[:])), reflect.ValueOf(func(dest interface{}, query string, args ...interface{}) { // selectCache // Prepare new one, if old one doesn't fit anymore. From 086b3d3f07133ddac90e030bdf393a4bcf51c553 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 3 Dec 2021 13:17:37 +0100 Subject: [PATCH 043/103] cmd/ido2icingadb: sliceIdoHistory(): re-use memory --- cmd/ido2icingadb/misc.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 1590eaf9..ea5bf36d 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -101,7 +101,6 @@ func sliceIdoHistory(snapshot *sqlx.Tx, query string, args map[string]interface{ rowsPtr := vNewRows.Interface() vRows := vNewRows.Elem() onRowsArgs := [1]reflect.Value{vRows} - vZeroRows := reflect.Zero(tRows) if args == nil { args = map[string]interface{}{} @@ -131,7 +130,7 @@ func sliceIdoHistory(snapshot *sqlx.Tx, query string, args map[string]interface{ break } - vRows.Set(vZeroRows) + vRows.Set(vRows.Slice(0, 0)) args["checkpoint"] = checkpoint } } From 2d2cad167743b23b4bd47d86e46399cc665b562a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 3 Dec 2021 13:49:19 +0100 Subject: [PATCH 044/103] cmd/ido2icingadb: sliceIdoHistory(): reduce onRows call complexity --- cmd/ido2icingadb/cache.go | 33 +++++++---- cmd/ido2icingadb/main.go | 115 +++++++++++++++++++------------------- cmd/ido2icingadb/misc.go | 19 +++---- 3 files changed, 87 insertions(+), 80 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index d2671534..86bb35ae 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -5,6 +5,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/pkg/errors" "math" + "reflect" "strings" "time" ) @@ -33,6 +34,14 @@ var eventTimeCacheSchema = []string{ // // Similar for acknowledgements. (On non-recoverable errors the whole program exits.) func buildEventTimeCache(ht *historyType, idoColumns []string) { + type row = struct { + Id uint64 + EventTime int64 + EventTimeUsec uint32 + EventIsStart uint8 + ObjectId uint64 + } + chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func()) { var checkpoint struct { Cnt int64 @@ -52,13 +61,9 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" > :checkpoint ORDER BY xh."+ht.idoIdColumn+" LIMIT :bulk", nil, checkpoint.MaxId.Int64, // ... since we were interrupted: - func(idoRows []struct { - Id uint64 - EventTime int64 - EventTimeUsec uint32 - EventIsStart uint8 - ObjectId uint64 - }) (checkpoint interface{}) { + reflect.TypeOf((*row)(nil)).Elem(), + func(ir interface{}) (checkpoint interface{}) { + idoRows := ir.([]row) for _, idoRow := range idoRows { if idoRow.EventIsStart == 0 { // Ack/flapping end event. Get the start event time: @@ -145,6 +150,12 @@ var previousHardStateCacheSchema = []string{ // // Similar for notifications. (On non-recoverable errors the whole program exits.) func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { + type row = struct { + Id uint64 + ObjectId uint64 + LastHardState uint8 + } + chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func()) { var nextIds struct { Cnt int64 @@ -188,11 +199,9 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" < :checkpoint ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT :bulk", nil, checkpoint, // ... since we were interrupted: - func(idoRows []struct { - Id uint64 - ObjectId uint64 - LastHardState uint8 - }) (checkpoint interface{}) { + reflect.TypeOf((*row)(nil)).Elem(), + func(ir interface{}) (checkpoint interface{}) { + idoRows := ir.([]row) for _, idoRow := range idoRows { var nhs []struct{ NextHardState uint8 } cacheSelect(*tx, &nhs, "SELECT next_hard_state FROM next_hard_state WHERE object_id=?", idoRow.ObjectId) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index a8f68e59..2b6024df 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -349,74 +349,73 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { // Stream IDO rows, ... sliceIdoHistory( ht.snapshot, ht.migrationQuery, args, ht.lastId, - reflect.MakeFunc(reflect.FuncOf([]reflect.Type{tRows}, []reflect.Type{tAny}, false), - func(args []reflect.Value) []reflect.Value { - vConvertRowsArgs[5] = args[0] // pass the rows + tRows.Elem(), + func(idoRows interface{}) (checkpoint interface{}) { + vConvertRowsArgs[5] = reflect.ValueOf(idoRows) // pass the rows - // ... convert them, ... - res := vConvertRows.Call(vConvertRowsArgs[:]) + // ... convert them, ... + res := vConvertRows.Call(vConvertRowsArgs[:]) - // ... and insert them: + // ... and insert them: - tx, err := idb.Beginx() - if err != nil { - log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) - } + tx, err := idb.Beginx() + if err != nil { + log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) + } - for _, operation := range [...]struct { - data reflect.Value - buildStmt func(subject interface{}) (stmt string, _ int) - stmtCache map[reflect.Type]string - }{{res[1], idb.BuildUpsertStmt, icingaDbInserts}, {res[0], idb.BuildUpdateStmt, icingaDbUpdates}} { - for _, table := range operation.data.Interface().([][]interface{}) { - if len(table) < 1 { - continue - } - - tRow := reflect.TypeOf(table[0]) - - query, ok := operation.stmtCache[tRow] - if !ok { - query, _ = operation.buildStmt(table[0]) - operation.stmtCache[tRow] = query - } - - stmt, err := tx.PrepareNamed(query) - if err != nil { - log.With("backend", "Icinga DB", "dml", query). - Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) - } - - for _, row := range table { - if _, err := stmt.Exec(row); err != nil { - log.With("backend", "Icinga DB", "dml", query, "args", row). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) - } - } - - _ = stmt.Close() + for _, operation := range [...]struct { + data reflect.Value + buildStmt func(subject interface{}) (stmt string, _ int) + stmtCache map[reflect.Type]string + }{{res[1], idb.BuildUpsertStmt, icingaDbInserts}, {res[0], idb.BuildUpdateStmt, icingaDbUpdates}} { + for _, table := range operation.data.Interface().([][]interface{}) { + if len(table) < 1 { + continue } - } - if lastIdoId := res[2].Interface(); lastIdoId != nil { - const stmt = "UPDATE ido_migration_progress SET last_ido_id=:last_ido_id " + - "WHERE history_type=:history_type" + tRow := reflect.TypeOf(table[0]) - args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} - if _, err := tx.NamedExec(stmt, args); err != nil { - log.With("backend", "Icinga DB", "dml", stmt, "args", args). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + query, ok := operation.stmtCache[tRow] + if !ok { + query, _ = operation.buildStmt(table[0]) + operation.stmtCache[tRow] = query } - } - if err := tx.Commit(); err != nil { - log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) - } + stmt, err := tx.PrepareNamed(query) + if err != nil { + log.With("backend", "Icinga DB", "dml", query). + Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) + } - inc.inc(args[0].Len()) - return res[2:] - }, - ).Interface(), + for _, row := range table { + if _, err := stmt.Exec(row); err != nil { + log.With("backend", "Icinga DB", "dml", query, "args", row). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + + _ = stmt.Close() + } + } + + if lastIdoId := res[2].Interface(); lastIdoId != nil { + const stmt = "UPDATE ido_migration_progress SET last_ido_id=:last_ido_id " + + "WHERE history_type=:history_type" + + args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} + if _, err := tx.NamedExec(stmt, args); err != nil { + log.With("backend", "Icinga DB", "dml", stmt, "args", args). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + + if err := tx.Commit(); err != nil { + log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) + } + + inc.inc(vConvertRowsArgs[5].Len()) + return res[2].Interface() + }, ) ht.bar.SetTotal(ht.bar.Current(), true) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index ea5bf36d..e94f9c5c 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -86,21 +86,20 @@ func calcServiceId(env, name1, name2 string) []byte { } // sliceIdoHistory performs query with args+map[string]interface{}{"checkpoint": checkpoint, "bulk": bulk} on snapshot -// and passes the results to onRows (a func([]T)interface{}) until either an empty result set or onRows() returns nil. +// and passes the results to onRows (a func([]rowType)interface{}) +// until either an empty result set or onRows() returns nil. // Rationale: split the likely large result set of a query by adding a WHERE condition and a LIMIT, // both with :named placeholders (:checkpoint, :bulk). // checkpoint is the initial value for the WHERE condition, onRows() returns follow-up ones. // (On non-recoverable errors the whole program exits.) -func sliceIdoHistory(snapshot *sqlx.Tx, query string, args map[string]interface{}, checkpoint, onRows interface{}) { - vOnRows := reflect.ValueOf(onRows) // TODO: make onRows generic[T] one nice day - - tRows := vOnRows.Type(). // func(rows []T) (checkpoint interface{}) - In(0) // []T - - vNewRows := reflect.New(tRows) +// TODO: make onRows generic[T] one nice day +func sliceIdoHistory( + snapshot *sqlx.Tx, query string, args map[string]interface{}, checkpoint interface{}, + rowType reflect.Type, onRows func(idoRows interface{}) (checkpoint interface{}), +) { + vNewRows := reflect.New(reflect.SliceOf(rowType)) rowsPtr := vNewRows.Interface() vRows := vNewRows.Elem() - onRowsArgs := [1]reflect.Value{vRows} if args == nil { args = map[string]interface{}{} @@ -126,7 +125,7 @@ func sliceIdoHistory(snapshot *sqlx.Tx, query string, args map[string]interface{ break } - if checkpoint = vOnRows.Call(onRowsArgs[:])[0].Interface(); checkpoint == nil { + if checkpoint = onRows(vRows.Interface()); checkpoint == nil { break } From 597cd631643a849f70227b296cfebddc7a8ecf8d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 3 Dec 2021 15:40:20 +0100 Subject: [PATCH 045/103] cmd/ido2icingadb: reduce historyType#convertRows call complexity --- cmd/ido2icingadb/convert.go | 20 +++++++--- cmd/ido2icingadb/main.go | 80 ++++++++++++++----------------------- cmd/ido2icingadb/misc.go | 13 +++++- 3 files changed, 55 insertions(+), 58 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index e2e0e3a7..d82582e7 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -55,8 +55,9 @@ type acknowledgementRow = struct { func convertAcknowledgementRows( env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []acknowledgementRow, + selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, ir interface{}, ) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) { + idoRows := ir.([]acknowledgementRow) if len(idoRows) < 1 { return } @@ -208,9 +209,11 @@ type commentRow = struct { func convertCommentRows( env string, envId, endpointId icingadbTypes.Binary, - _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []commentRow, + _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, ir interface{}, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { + idoRows := ir.([]commentRow) var commentHistory, allHistory []interface{} + for _, row := range idoRows { id := calcObjectId(env, row.Name) typ := objectTypes[row.ObjecttypeId] @@ -326,9 +329,11 @@ type downtimeRow = struct { func convertDowntimeRows( env string, envId, endpointId icingadbTypes.Binary, - _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []downtimeRow, + _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, ir interface{}, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { + idoRows := ir.([]downtimeRow) var downtimeHistory, allHistory []interface{} + for _, row := range idoRows { id := calcObjectId(env, row.Name) typ := objectTypes[row.ObjecttypeId] @@ -475,8 +480,9 @@ type flappingRow = struct { func convertFlappingRows( env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []flappingRow, + selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, ir interface{}, ) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) { + idoRows := ir.([]flappingRow) if len(idoRows) < 1 { return } @@ -625,8 +631,9 @@ type notificationRow = struct { func convertNotificationRows( env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []notificationRow, + selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, ir interface{}, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { + idoRows := ir.([]notificationRow) if len(idoRows) < 1 { return } @@ -802,8 +809,9 @@ type stateRow = struct { func convertStateRows( env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []stateRow, + selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, ir interface{}, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { + idoRows := ir.([]stateRow) if len(idoRows) < 1 { return } diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 2b6024df..70552a2f 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -9,7 +9,6 @@ import ( "github.com/goccy/go-yaml" "github.com/icinga/icingadb/pkg/config" "github.com/icinga/icingadb/pkg/icingadb" - icingadbTypes "github.com/icinga/icingadb/pkg/types" "github.com/jessevdk/go-flags" "github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx/reflectx" @@ -267,10 +266,6 @@ func fillCache() { progress.Wait() } -// tAny can't be created by just using reflect.TypeOf(interface{}(nil)). -var tAny = reflect.TypeOf((*interface{})(nil)). // *interface{} - Elem() // interface{} - // migrate does the actual migration. func migrate(c *Config, idb *icingadb.DB, envId []byte) { endpointId := sha1.Sum([]byte(c.Icinga2.Endpoint)) @@ -281,16 +276,6 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { } types.forEach(func(ht *historyType) { - // type rowStructPtr = interface{} - // type table = []rowStructPtr - // func convertRowsFromIdoToIcingaDb(env string, envId, endpointId Binary, - // selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []T) - // (icingaDbUpdates, icingaDbInserts []table, checkpoint interface{}) - vConvertRows := reflect.ValueOf(ht.convertRows) // TODO: make historyType#convertRows generic[T] one nice day - - // []T (idoRows) - tRows := vConvertRows.Type().In(5) - var lastQuery string var lastStmt *sqlx.Stmt @@ -300,34 +285,28 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { } }() - vConvertRowsArgs := [6]reflect.Value{ - reflect.ValueOf(c.Icinga2.Env), reflect.ValueOf(icingadbTypes.Binary(envId)), - reflect.ValueOf(icingadbTypes.Binary(endpointId[:])), - reflect.ValueOf(func(dest interface{}, query string, args ...interface{}) { // selectCache - // Prepare new one, if old one doesn't fit anymore. - if query != lastQuery { - if lastStmt != nil { - _ = lastStmt.Close() - } - - var err error - - lastStmt, err = ht.cache.Preparex(query) - if err != nil { - log.With("backend", "cache", "query", query). - Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - - lastQuery = query + selectCache := func(dest interface{}, query string, args ...interface{}) { + // Prepare new one, if old one doesn't fit anymore. + if query != lastQuery { + if lastStmt != nil { + _ = lastStmt.Close() } - if err := lastStmt.Select(dest, args...); err != nil { - log.With("backend", "cache", "query", query, "args", args). - Fatalf("%+v", errors.Wrap(err, "can't perform query")) + var err error + + lastStmt, err = ht.cache.Preparex(query) + if err != nil { + log.With("backend", "cache", "query", query). + Fatalf("%+v", errors.Wrap(err, "can't prepare query")) } - }), - reflect.ValueOf(ht.snapshot), - // and the rows (below) + + lastQuery = query + } + + if err := lastStmt.Select(dest, args...); err != nil { + log.With("backend", "cache", "query", query, "args", args). + Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } } var args map[string]interface{} @@ -348,13 +327,12 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { // Stream IDO rows, ... sliceIdoHistory( - ht.snapshot, ht.migrationQuery, args, ht.lastId, - tRows.Elem(), + ht.snapshot, ht.migrationQuery, args, ht.lastId, ht.convertRowType, func(idoRows interface{}) (checkpoint interface{}) { - vConvertRowsArgs[5] = reflect.ValueOf(idoRows) // pass the rows - // ... convert them, ... - res := vConvertRows.Call(vConvertRowsArgs[:]) + updates, inserts, lastIdoId := ht.convertRows( + c.Icinga2.Env, envId, endpointId[:], selectCache, ht.snapshot, idoRows, + ) // ... and insert them: @@ -364,11 +342,11 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { } for _, operation := range [...]struct { - data reflect.Value + data [][]interface{} buildStmt func(subject interface{}) (stmt string, _ int) stmtCache map[reflect.Type]string - }{{res[1], idb.BuildUpsertStmt, icingaDbInserts}, {res[0], idb.BuildUpdateStmt, icingaDbUpdates}} { - for _, table := range operation.data.Interface().([][]interface{}) { + }{{inserts, idb.BuildUpsertStmt, icingaDbInserts}, {updates, idb.BuildUpdateStmt, icingaDbUpdates}} { + for _, table := range operation.data { if len(table) < 1 { continue } @@ -398,7 +376,7 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { } } - if lastIdoId := res[2].Interface(); lastIdoId != nil { + if lastIdoId != nil { const stmt = "UPDATE ido_migration_progress SET last_ido_id=:last_ido_id " + "WHERE history_type=:history_type" @@ -413,8 +391,8 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } - inc.inc(vConvertRowsArgs[5].Len()) - return res[2].Interface() + inc.inc(reflect.ValueOf(idoRows).Len()) + return lastIdoId }, ) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index e94f9c5c..98bf53fd 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -150,8 +150,13 @@ type historyType struct { cacheLimitQuery string // migrationQuery SELECTs source data for actual migration. migrationQuery string + // convertRowType specifies convertRows' idoRows. + convertRowType reflect.Type // convertRows intention: see migrate(). - convertRows interface{} + // TODO: make idoRows generic[T] one nice day + convertRows func(env string, envId, endpointId icingadbTypes.Binary, + selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, + idoRows interface{}) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) // cache represents .sqlite3. cache *sqlx.DB @@ -209,6 +214,7 @@ var types = historyTypes{ }) }, migrationQuery: acknowledgementMigrationQuery, + convertRowType: reflect.TypeOf((*acknowledgementRow)(nil)).Elem(), convertRows: convertAcknowledgementRows, }, { @@ -216,6 +222,7 @@ var types = historyTypes{ idoTable: "icinga_commenthistory", idoIdColumn: "commenthistory_id", migrationQuery: commentMigrationQuery, + convertRowType: reflect.TypeOf((*commentRow)(nil)).Elem(), convertRows: convertCommentRows, }, { @@ -223,6 +230,7 @@ var types = historyTypes{ idoTable: "icinga_downtimehistory", idoIdColumn: "downtimehistory_id", migrationQuery: downtimeMigrationQuery, + convertRowType: reflect.TypeOf((*downtimeRow)(nil)).Elem(), convertRows: convertDowntimeRows, }, { @@ -237,6 +245,7 @@ var types = historyTypes{ }) }, migrationQuery: flappingMigrationQuery, + convertRowType: reflect.TypeOf((*flappingRow)(nil)).Elem(), convertRows: convertFlappingRows, }, { @@ -251,6 +260,7 @@ var types = historyTypes{ }, cacheLimitQuery: "SELECT MAX(history_id) FROM previous_hard_state", migrationQuery: notificationMigrationQuery, + convertRowType: reflect.TypeOf((*notificationRow)(nil)).Elem(), convertRows: convertNotificationRows, }, { @@ -263,6 +273,7 @@ var types = historyTypes{ }, cacheLimitQuery: "SELECT MAX(history_id) FROM previous_hard_state", migrationQuery: stateMigrationQuery, + convertRowType: reflect.TypeOf((*stateRow)(nil)).Elem(), convertRows: convertStateRows, }, } From a2dae38b00b4c63313af9c372b1a410bc60cb73e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 3 Dec 2021 15:56:12 +0100 Subject: [PATCH 046/103] cmd/ido2icingadb: build fix --- cmd/ido2icingadb/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 70552a2f..236defe8 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -9,6 +9,7 @@ import ( "github.com/goccy/go-yaml" "github.com/icinga/icingadb/pkg/config" "github.com/icinga/icingadb/pkg/icingadb" + "github.com/icinga/icingadb/pkg/logging" "github.com/jessevdk/go-flags" "github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx/reflectx" @@ -161,7 +162,7 @@ func connectAll(c *Config) (ido, idb *icingadb.DB) { // connect connects to which DB as cfg specifies. (On non-recoverable errors the whole program exits.) func connect(which string, cfg *config.Database) *icingadb.DB { - db, err := cfg.Open(log) + db, err := cfg.Open(logging.NewLogger(log, 20*time.Second)) if err != nil { log.With("backend", which).Fatalf("%+v", errors.Wrap(err, "can't connect to database")) } From aa585e9d7251a646471e3ea34b0bf52dbc6eb60b Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 1 Mar 2022 12:02:28 +0100 Subject: [PATCH 047/103] cmd/ido2icingadb: don't hash non-hashable types --- cmd/ido2icingadb/convert.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index d82582e7..dceee4e0 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -708,8 +708,9 @@ func convertNotificationRows( } ts := convertTime(row.EndTime, row.EndTimeUsec) - notificationHistoryId := hashAny([]interface{}{env, name, ntEnum, ts}) - id := hashAny([]interface{}{env, "notification", name, ntEnum, ts}) + tsMilli := float64(utils.UnixMilli(ts.Time())) + notificationHistoryId := hashAny([]interface{}{env, name, ntEnum, tsMilli}) + id := hashAny([]interface{}{env, "notification", name, ntEnum, tsMilli}) typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) serviceId := calcServiceId(env, row.Name1, row.Name2) @@ -839,8 +840,9 @@ func convertStateRows( name := strings.Join([]string{row.Name1, row.Name2}, "!") ts := convertTime(row.StateTime, row.StateTimeUsec) - stateHistoryId := hashAny([]interface{}{env, name, ts}) - id := hashAny([]interface{}{env, "state_change", name, ts}) + tsMilli := float64(utils.UnixMilli(ts.Time())) + stateHistoryId := hashAny([]interface{}{env, name, tsMilli}) + id := hashAny([]interface{}{env, "state_change", name, tsMilli}) typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) serviceId := calcServiceId(env, row.Name1, row.Name2) From bcf22567960bd6f9f28e9d94030bbf44286e55dc Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 1 Mar 2022 13:28:09 +0100 Subject: [PATCH 048/103] cmd/ido2icingadb: correct mismatching FK ID --- cmd/ido2icingadb/convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index dceee4e0..dc8dacd4 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -767,7 +767,7 @@ func convertNotificationRows( }, }, EnvironmentMeta: v1.EnvironmentMeta{EnvironmentId: envId}, - NotificationHistoryId: id, + NotificationHistoryId: notificationHistoryId, UserId: userId, }) } From d9a3e045370c03cf79df3472d74d877c990a13f7 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 10 Mar 2022 11:15:13 +0100 Subject: [PATCH 049/103] cmd/ido2icingadb: migrate(): run only one transaction at a time to avoid inter-transaction lock timeouts. --- cmd/ido2icingadb/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 236defe8..54553f62 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -20,6 +20,7 @@ import ( "os" "path" "reflect" + "sync" "time" ) @@ -270,6 +271,7 @@ func fillCache() { // migrate does the actual migration. func migrate(c *Config, idb *icingadb.DB, envId []byte) { endpointId := sha1.Sum([]byte(c.Icinga2.Endpoint)) + idbTx := &sync.Mutex{} progress := mpb.New() for i := range types { @@ -337,6 +339,9 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { // ... and insert them: + idbTx.Lock() + defer idbTx.Unlock() + tx, err := idb.Beginx() if err != nil { log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) From 581270ffee1d7cec9bc9ca7819b8b64834ce8835 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 17 Mar 2022 13:02:14 +0100 Subject: [PATCH 050/103] cmd/ido2icingadb: support Postgres --- cmd/ido2icingadb/convert.go | 41 +++++++++++++++---------- cmd/ido2icingadb/main.go | 40 ++++++++++++++---------- cmd/ido2icingadb/misc.go | 27 ++++++++++++++++ pkg/icingadb/v1/history/notification.go | 2 +- 4 files changed, 75 insertions(+), 35 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index dc8dacd4..ade0076a 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -17,7 +17,7 @@ import ( const acknowledgementMigrationQuery = "SELECT ah.acknowledgement_id, UNIX_TIMESTAMP(ah.entry_time) entry_time, " + "ah.entry_time_usec, ah.acknowledgement_type, ah.author_name, ah.comment_data, ah.is_sticky, " + "ah.persistent_comment, UNIX_TIMESTAMP(ah.end_time) end_time, o.objecttype_id, o.name1, " + - "IFNULL(o.name2, '') name2 " + + "COALESCE(o.name2, '') name2 " + "FROM icinga_acknowledgements ah USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=ah.object_id " + "WHERE ah.acknowledgement_id > :checkpoint " + // where we were interrupted @@ -181,9 +181,9 @@ func convertAcknowledgementRows( const commentMigrationQuery = "SELECT ch.commenthistory_id, UNIX_TIMESTAMP(ch.entry_time) entry_time, " + "ch.entry_time_usec, ch.entry_type, ch.author_name, ch.comment_data, ch.is_persistent, " + - "IFNULL(UNIX_TIMESTAMP(ch.expiration_time), 0) expiration_time, " + - "IFNULL(UNIX_TIMESTAMP(ch.deletion_time), 0) deletion_time, ch.deletion_time_usec, ch.name, " + - "o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + + "COALESCE(UNIX_TIMESTAMP(ch.expiration_time), 0) expiration_time, " + + "COALESCE(UNIX_TIMESTAMP(ch.deletion_time), 0) deletion_time, ch.deletion_time_usec, ch.name, " + + "o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 " + "FROM icinga_commenthistory ch USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=ch.object_id " + "WHERE ch.commenthistory_id > :checkpoint " + // where we were interrupted @@ -293,11 +293,11 @@ func convertCommentRows( const downtimeMigrationQuery = "SELECT dh.downtimehistory_id, UNIX_TIMESTAMP(dh.entry_time) entry_time, " + "dh.author_name, dh.comment_data, dh.is_fixed, dh.duration, " + "UNIX_TIMESTAMP(dh.scheduled_start_time) scheduled_start_time, " + - "IFNULL(UNIX_TIMESTAMP(dh.scheduled_end_time), 0) scheduled_end_time, " + - "IFNULL(UNIX_TIMESTAMP(dh.actual_start_time), 0) actual_start_time, dh.actual_start_time_usec, " + - "IFNULL(UNIX_TIMESTAMP(dh.actual_end_time), 0) actual_end_time, dh.actual_end_time_usec, dh.was_cancelled, " + - "IFNULL(UNIX_TIMESTAMP(dh.trigger_time), 0) trigger_time, dh.name, o.objecttype_id, o.name1, " + - "IFNULL(o.name2, '') name2, IFNULL(sd.name, '') triggered_by " + + "COALESCE(UNIX_TIMESTAMP(dh.scheduled_end_time), 0) scheduled_end_time, " + + "COALESCE(UNIX_TIMESTAMP(dh.actual_start_time), 0) actual_start_time, dh.actual_start_time_usec, " + + "COALESCE(UNIX_TIMESTAMP(dh.actual_end_time), 0) actual_end_time, dh.actual_end_time_usec, dh.was_cancelled, " + + "COALESCE(UNIX_TIMESTAMP(dh.trigger_time), 0) trigger_time, dh.name, o.objecttype_id, o.name1, " + + "COALESCE(o.name2, '') name2, COALESCE(sd.name, '') triggered_by " + "FROM icinga_downtimehistory dh USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=dh.object_id " + "LEFT JOIN icinga_scheduleddowntime sd ON sd.scheduleddowntime_id=dh.triggered_by_id " + @@ -444,7 +444,7 @@ func convertDowntimeRows( const flappingMigrationQuery = "SELECT fh.flappinghistory_id, UNIX_TIMESTAMP(fh.event_time) event_time, " + "fh.event_time_usec, fh.event_type, fh.percent_state_change, fh.low_threshold, " + - "fh.high_threshold, o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + + "fh.high_threshold, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 " + "FROM icinga_flappinghistory fh USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=fh.object_id " + "WHERE fh.flappinghistory_id > :checkpoint " + // where we were interrupted @@ -603,8 +603,8 @@ func convertFlappingRows( } const notificationMigrationQuery = "SELECT n.notification_id, n.notification_reason, " + - "UNIX_TIMESTAMP(n.end_time) end_time, n.end_time_usec, n.state, IFNULL(n.output, '') output, " + - "n.long_output, n.contacts_notified, o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + + "UNIX_TIMESTAMP(n.end_time) end_time, n.end_time_usec, n.state, COALESCE(n.output, '') output, " + + "n.long_output, n.contacts_notified, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 " + "FROM icinga_notifications n USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=n.object_id " + "WHERE n.notification_id <= :cache_limit AND " + @@ -658,8 +658,10 @@ func convertNotificationRows( } { - const query = "SELECT c.notification_id, o.name1 FROM icinga_contactnotifications c " + - "INNER JOIN icinga_objects o ON o.object_id=c.contact_object_id WHERE c.notification_id BETWEEN ? AND ?" + var query = ido.Rebind( + "SELECT c.notification_id, o.name1 FROM icinga_contactnotifications c " + + "INNER JOIN icinga_objects o ON o.object_id=c.contact_object_id WHERE c.notification_id BETWEEN ? AND ?", + ) err := ido.Select(&contacts, query, idoRows[0].NotificationId, idoRows[len(idoRows)-1].NotificationId) if err != nil { @@ -739,8 +741,13 @@ func convertNotificationRows( State: row.State, PreviousHardState: previousHardState, Author: "-", - Text: text, - UsersNotified: row.ContactsNotified, + Text: icingadbTypes.String{ + NullString: sql.NullString{ + String: text, + Valid: true, + }, + }, + UsersNotified: row.ContactsNotified, }) allHistory = append(allHistory, &history.HistoryNotification{ @@ -782,7 +789,7 @@ func convertNotificationRows( const stateMigrationQuery = "SELECT sh.statehistory_id, UNIX_TIMESTAMP(sh.state_time) state_time, " + "sh.state_time_usec, sh.state, sh.state_type, sh.current_check_attempt, " + "sh.max_check_attempts, sh.last_state, sh.last_hard_state, sh.output, sh.long_output, " + - "sh.check_source, o.objecttype_id, o.name1, IFNULL(o.name2, '') name2 " + + "sh.check_source, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 " + "FROM icinga_statehistory sh USE INDEX (PRIMARY) " + "INNER JOIN icinga_objects o ON o.object_id=sh.object_id " + "WHERE sh.statehistory_id <= :cache_limit AND " + diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 54553f62..03b6611b 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/goccy/go-yaml" "github.com/icinga/icingadb/pkg/config" + "github.com/icinga/icingadb/pkg/driver" "github.com/icinga/icingadb/pkg/icingadb" "github.com/icinga/icingadb/pkg/logging" "github.com/jessevdk/go-flags" @@ -193,26 +194,27 @@ func startIdoTx(ido *icingadb.DB) { func computeProgress(idb *icingadb.DB) { { _, err := idb.Exec(`CREATE TABLE IF NOT EXISTS ido_migration_progress ( - history_type VARCHAR(63) PRIMARY KEY, - last_ido_id BIGINT unsigned NOT NULL + history_type VARCHAR(63) NOT NULL, + last_ido_id BIGINT NOT NULL, + + CONSTRAINT pk_ido_migration_progress PRIMARY KEY (history_type) )`) if err != nil { log.Fatalf("%+v", errors.Wrap(err, "can't create table ido_migration_progress")) } } - types.forEach(func(ht *historyType) { - stmt := idb.Rebind( - "INSERT INTO ido_migration_progress(history_type, last_ido_id) " + - "VALUES (?, 0) ON DUPLICATE KEY UPDATE history_type=history_type", - ) + stmt, _ := idb.BuildUpsertStmt(&IdoMigrationProgress{}) - if _, err := idb.Exec(stmt, ht.name); err != nil { - log.With("backend", "Icinga DB", "dml", stmt, "args", []interface{}{ht.name}). + types.forEach(func(ht *historyType) { + row := &IdoMigrationProgress{IdoMigrationProgressUpserter{ht.name}, 0} + + if _, err := idb.NamedExec(stmt, row); err != nil { + log.With("backend", "Icinga DB", "dml", stmt, "args", []interface{}{ht.name, 0}). Fatalf("%+v", errors.Wrap(err, "can't perform DML")) } - const query = "SELECT last_ido_id FROM ido_migration_progress WHERE history_type=?" + var query = idb.Rebind("SELECT last_ido_id FROM ido_migration_progress WHERE history_type=?") if err := idb.Get(&ht.lastId, query, ht.name); err != nil { log.With("backend", "Icinga DB", "query", query, "args", []interface{}{ht.name}). @@ -228,11 +230,13 @@ func computeProgress(idb *icingadb.DB) { err := ht.snapshot.Select( &rows, - // For actual migration icinga_objects will be joined anyway, - // so it makes no sense to take vanished objects into account. - "SELECT xh."+ht.idoIdColumn+"<=? migrated, COUNT(*) cnt FROM "+ht.idoTable+" xh"+ - " INNER JOIN icinga_objects o ON o.object_id=xh.object_id GROUP BY xh."+ht.idoIdColumn+"<=?", - ht.lastId, ht.lastId, + ht.snapshot.Rebind( + // For actual migration icinga_objects will be joined anyway, + // so it makes no sense to take vanished objects into account. + "SELECT CASE WHEN xh."+ht.idoIdColumn+"<=? THEN 1 ELSE 0 END migrated, COUNT(*) cnt FROM "+ + ht.idoTable+" xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id GROUP BY migrated", + ), + ht.lastId, ) if err != nil { log.Fatalf("%+v", errors.Wrap(err, "can't count query")) @@ -339,8 +343,10 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { // ... and insert them: - idbTx.Lock() - defer idbTx.Unlock() + if idb.DriverName() == driver.MySQL { + idbTx.Lock() + defer idbTx.Unlock() + } tx, err := idb.Beginx() if err != nil { diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 98bf53fd..67aa5043 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -3,6 +3,8 @@ package main import ( "context" "crypto/sha1" + "github.com/icinga/icingadb/pkg/contracts" + "github.com/icinga/icingadb/pkg/driver" "github.com/icinga/icingadb/pkg/icingadb/objectpacker" icingadbTypes "github.com/icinga/icingadb/pkg/types" "github.com/jmoiron/sqlx" @@ -12,6 +14,7 @@ import ( "go.uber.org/zap" "golang.org/x/sync/errgroup" "reflect" + "strings" "time" ) @@ -33,6 +36,26 @@ func (bi *barIncrementer) inc(i int) { bi.bar.DecoratorEwmaUpdate(now.Sub(prev)) } +type IdoMigrationProgressUpserter struct { + HistoryType string `json:"history_type"` +} + +// Upsert implements the contracts.Upserter interface. +func (impu *IdoMigrationProgressUpserter) Upsert() interface{} { + return impu +} + +type IdoMigrationProgress struct { + IdoMigrationProgressUpserter `json:",inline"` + LastIdoId uint64 `json:"last_ido_id"` +} + +// Assert interface compliance. +var ( + _ contracts.Upserter = (*IdoMigrationProgressUpserter)(nil) + _ contracts.Upserter = (*IdoMigrationProgress)(nil) +) + const bulk = 10000 // log is the root logger. @@ -108,6 +131,10 @@ func sliceIdoHistory( args["checkpoint"] = checkpoint args["bulk"] = bulk + if snapshot.DriverName() != driver.MySQL { + query = strings.ReplaceAll(query, " USE INDEX (PRIMARY)", "") + } + for { // TODO: use Tx#SelectNamed() one nice day (https://github.com/jmoiron/sqlx/issues/779) stmt, err := snapshot.PrepareNamed(query) diff --git a/pkg/icingadb/v1/history/notification.go b/pkg/icingadb/v1/history/notification.go index 1ebc27aa..17fd375c 100644 --- a/pkg/icingadb/v1/history/notification.go +++ b/pkg/icingadb/v1/history/notification.go @@ -15,7 +15,7 @@ type NotificationHistory struct { State uint8 `json:"state"` PreviousHardState uint8 `json:"previous_hard_state"` Author string `json:"author"` - Text string `json:"text"` + Text types.String `json:"text"` UsersNotified uint16 `json:"users_notified"` } From f522b2923adc6d5c3876499771c309274f707f26 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 4 Apr 2022 18:54:21 +0200 Subject: [PATCH 051/103] cmd/ido2icingadb: prefer generics over reflection --- cmd/ido2icingadb/cache.go | 9 +- cmd/ido2icingadb/convert.go | 18 +-- cmd/ido2icingadb/main.go | 263 +++++++++++++++++++----------------- cmd/ido2icingadb/misc.go | 65 +++++---- 4 files changed, 176 insertions(+), 179 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 86bb35ae..3e06ec3d 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -5,7 +5,6 @@ import ( "github.com/jmoiron/sqlx" "github.com/pkg/errors" "math" - "reflect" "strings" "time" ) @@ -61,9 +60,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" > :checkpoint ORDER BY xh."+ht.idoIdColumn+" LIMIT :bulk", nil, checkpoint.MaxId.Int64, // ... since we were interrupted: - reflect.TypeOf((*row)(nil)).Elem(), - func(ir interface{}) (checkpoint interface{}) { - idoRows := ir.([]row) + func(idoRows []row) (checkpoint interface{}) { for _, idoRow := range idoRows { if idoRow.EventIsStart == 0 { // Ack/flapping end event. Get the start event time: @@ -199,9 +196,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ ht.idoIdColumn+" < :checkpoint ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT :bulk", nil, checkpoint, // ... since we were interrupted: - reflect.TypeOf((*row)(nil)).Elem(), - func(ir interface{}) (checkpoint interface{}) { - idoRows := ir.([]row) + func(idoRows []row) (checkpoint interface{}) { for _, idoRow := range idoRows { var nhs []struct{ NextHardState uint8 } cacheSelect(*tx, &nhs, "SELECT next_hard_state FROM next_hard_state WHERE object_id=?", idoRow.ObjectId) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index ade0076a..d36ca06a 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -55,9 +55,8 @@ type acknowledgementRow = struct { func convertAcknowledgementRows( env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, ir interface{}, + selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []acknowledgementRow, ) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) { - idoRows := ir.([]acknowledgementRow) if len(idoRows) < 1 { return } @@ -209,9 +208,8 @@ type commentRow = struct { func convertCommentRows( env string, envId, endpointId icingadbTypes.Binary, - _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, ir interface{}, + _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []commentRow, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { - idoRows := ir.([]commentRow) var commentHistory, allHistory []interface{} for _, row := range idoRows { @@ -329,9 +327,8 @@ type downtimeRow = struct { func convertDowntimeRows( env string, envId, endpointId icingadbTypes.Binary, - _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, ir interface{}, + _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []downtimeRow, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { - idoRows := ir.([]downtimeRow) var downtimeHistory, allHistory []interface{} for _, row := range idoRows { @@ -480,9 +477,8 @@ type flappingRow = struct { func convertFlappingRows( env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, ir interface{}, + selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []flappingRow, ) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) { - idoRows := ir.([]flappingRow) if len(idoRows) < 1 { return } @@ -631,9 +627,8 @@ type notificationRow = struct { func convertNotificationRows( env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, ir interface{}, + selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []notificationRow, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { - idoRows := ir.([]notificationRow) if len(idoRows) < 1 { return } @@ -817,9 +812,8 @@ type stateRow = struct { func convertStateRows( env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, ir interface{}, + selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []stateRow, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { - idoRows := ir.([]stateRow) if len(idoRows) < 1 { return } diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 03b6611b..55661889 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -11,6 +11,7 @@ import ( "github.com/icinga/icingadb/pkg/driver" "github.com/icinga/icingadb/pkg/icingadb" "github.com/icinga/icingadb/pkg/logging" + icingadbTypes "github.com/icinga/icingadb/pkg/types" "github.com/jessevdk/go-flags" "github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx/reflectx" @@ -283,133 +284,143 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { } types.forEach(func(ht *historyType) { - var lastQuery string - var lastStmt *sqlx.Stmt - - defer func() { - if lastStmt != nil { - _ = lastStmt.Close() - } - }() - - selectCache := func(dest interface{}, query string, args ...interface{}) { - // Prepare new one, if old one doesn't fit anymore. - if query != lastQuery { - if lastStmt != nil { - _ = lastStmt.Close() - } - - var err error - - lastStmt, err = ht.cache.Preparex(query) - if err != nil { - log.With("backend", "cache", "query", query). - Fatalf("%+v", errors.Wrap(err, "can't prepare query")) - } - - lastQuery = query - } - - if err := lastStmt.Select(dest, args...); err != nil { - log.With("backend", "cache", "query", query, "args", args). - Fatalf("%+v", errors.Wrap(err, "can't perform query")) - } - } - - var args map[string]interface{} - - // For the case that the cache was older that the IDO, - // but ht.cacheFiller couldn't update it, limit (WHERE) our source data set. - if ht.cacheLimitQuery != "" { - var limit uint64 - cacheGet(ht.cache, &limit, ht.cacheLimitQuery) - args = map[string]interface{}{"cache_limit": limit} - } - - icingaDbInserts := map[reflect.Type]string{} - icingaDbUpdates := map[reflect.Type]string{} - - ht.bar.SetCurrent(ht.done) - inc := barIncrementer{ht.bar, time.Now()} - - // Stream IDO rows, ... - sliceIdoHistory( - ht.snapshot, ht.migrationQuery, args, ht.lastId, ht.convertRowType, - func(idoRows interface{}) (checkpoint interface{}) { - // ... convert them, ... - updates, inserts, lastIdoId := ht.convertRows( - c.Icinga2.Env, envId, endpointId[:], selectCache, ht.snapshot, idoRows, - ) - - // ... and insert them: - - if idb.DriverName() == driver.MySQL { - idbTx.Lock() - defer idbTx.Unlock() - } - - tx, err := idb.Beginx() - if err != nil { - log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) - } - - for _, operation := range [...]struct { - data [][]interface{} - buildStmt func(subject interface{}) (stmt string, _ int) - stmtCache map[reflect.Type]string - }{{inserts, idb.BuildUpsertStmt, icingaDbInserts}, {updates, idb.BuildUpdateStmt, icingaDbUpdates}} { - for _, table := range operation.data { - if len(table) < 1 { - continue - } - - tRow := reflect.TypeOf(table[0]) - - query, ok := operation.stmtCache[tRow] - if !ok { - query, _ = operation.buildStmt(table[0]) - operation.stmtCache[tRow] = query - } - - stmt, err := tx.PrepareNamed(query) - if err != nil { - log.With("backend", "Icinga DB", "dml", query). - Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) - } - - for _, row := range table { - if _, err := stmt.Exec(row); err != nil { - log.With("backend", "Icinga DB", "dml", query, "args", row). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) - } - } - - _ = stmt.Close() - } - } - - if lastIdoId != nil { - const stmt = "UPDATE ido_migration_progress SET last_ido_id=:last_ido_id " + - "WHERE history_type=:history_type" - - args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} - if _, err := tx.NamedExec(stmt, args); err != nil { - log.With("backend", "Icinga DB", "dml", stmt, "args", args). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) - } - } - - if err := tx.Commit(); err != nil { - log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) - } - - inc.inc(reflect.ValueOf(idoRows).Len()) - return lastIdoId - }, - ) - - ht.bar.SetTotal(ht.bar.Current(), true) + ht.migrate(c, idb, envId, endpointId, idbTx, ht) }) progress.Wait() } + +// migrate does the actual migration for one history type. +func migrateOneType[IdoRow any]( + c *Config, idb *icingadb.DB, envId []byte, endpointId [sha1.Size]byte, idbTx *sync.Mutex, ht *historyType, + convertRows func(env string, envId, endpointId icingadbTypes.Binary, + selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, + idoRows []IdoRow) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}), +) { + var lastQuery string + var lastStmt *sqlx.Stmt + + defer func() { + if lastStmt != nil { + _ = lastStmt.Close() + } + }() + + selectCache := func(dest interface{}, query string, args ...interface{}) { + // Prepare new one, if old one doesn't fit anymore. + if query != lastQuery { + if lastStmt != nil { + _ = lastStmt.Close() + } + + var err error + + lastStmt, err = ht.cache.Preparex(query) + if err != nil { + log.With("backend", "cache", "query", query). + Fatalf("%+v", errors.Wrap(err, "can't prepare query")) + } + + lastQuery = query + } + + if err := lastStmt.Select(dest, args...); err != nil { + log.With("backend", "cache", "query", query, "args", args). + Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + } + + var args map[string]interface{} + + // For the case that the cache was older that the IDO, + // but ht.cacheFiller couldn't update it, limit (WHERE) our source data set. + if ht.cacheLimitQuery != "" { + var limit uint64 + cacheGet(ht.cache, &limit, ht.cacheLimitQuery) + args = map[string]interface{}{"cache_limit": limit} + } + + icingaDbInserts := map[reflect.Type]string{} + icingaDbUpdates := map[reflect.Type]string{} + + ht.bar.SetCurrent(ht.done) + inc := barIncrementer{ht.bar, time.Now()} + + // Stream IDO rows, ... + sliceIdoHistory( + ht.snapshot, ht.migrationQuery, args, ht.lastId, + func(idoRows []IdoRow) (checkpoint interface{}) { + // ... convert them, ... + updates, inserts, lastIdoId := convertRows( + c.Icinga2.Env, envId, endpointId[:], selectCache, ht.snapshot, idoRows, + ) + + // ... and insert them: + + if idb.DriverName() == driver.MySQL { + idbTx.Lock() + defer idbTx.Unlock() + } + + tx, err := idb.Beginx() + if err != nil { + log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) + } + + for _, operation := range [...]struct { + data [][]interface{} + buildStmt func(subject interface{}) (stmt string, _ int) + stmtCache map[reflect.Type]string + }{{inserts, idb.BuildUpsertStmt, icingaDbInserts}, {updates, idb.BuildUpdateStmt, icingaDbUpdates}} { + for _, table := range operation.data { + if len(table) < 1 { + continue + } + + tRow := reflect.TypeOf(table[0]) + + query, ok := operation.stmtCache[tRow] + if !ok { + query, _ = operation.buildStmt(table[0]) + operation.stmtCache[tRow] = query + } + + stmt, err := tx.PrepareNamed(query) + if err != nil { + log.With("backend", "Icinga DB", "dml", query). + Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) + } + + for _, row := range table { + if _, err := stmt.Exec(row); err != nil { + log.With("backend", "Icinga DB", "dml", query, "args", row). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + + _ = stmt.Close() + } + } + + if lastIdoId != nil { + const stmt = "UPDATE ido_migration_progress SET last_ido_id=:last_ido_id " + + "WHERE history_type=:history_type" + + args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} + if _, err := tx.NamedExec(stmt, args); err != nil { + log.With("backend", "Icinga DB", "dml", stmt, "args", args). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + + if err := tx.Commit(); err != nil { + log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) + } + + inc.inc(len(idoRows)) + return lastIdoId + }, + ) + + ht.bar.SetTotal(ht.bar.Current(), true) +} diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 67aa5043..a3d2f287 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -5,6 +5,7 @@ import ( "crypto/sha1" "github.com/icinga/icingadb/pkg/contracts" "github.com/icinga/icingadb/pkg/driver" + "github.com/icinga/icingadb/pkg/icingadb" "github.com/icinga/icingadb/pkg/icingadb/objectpacker" icingadbTypes "github.com/icinga/icingadb/pkg/types" "github.com/jmoiron/sqlx" @@ -13,8 +14,8 @@ import ( "github.com/vbauerster/mpb/v6/decor" "go.uber.org/zap" "golang.org/x/sync/errgroup" - "reflect" "strings" + "sync" "time" ) @@ -109,21 +110,15 @@ func calcServiceId(env, name1, name2 string) []byte { } // sliceIdoHistory performs query with args+map[string]interface{}{"checkpoint": checkpoint, "bulk": bulk} on snapshot -// and passes the results to onRows (a func([]rowType)interface{}) -// until either an empty result set or onRows() returns nil. +// and passes the results to onRows until either an empty result set or onRows() returns nil. // Rationale: split the likely large result set of a query by adding a WHERE condition and a LIMIT, // both with :named placeholders (:checkpoint, :bulk). // checkpoint is the initial value for the WHERE condition, onRows() returns follow-up ones. // (On non-recoverable errors the whole program exits.) -// TODO: make onRows generic[T] one nice day -func sliceIdoHistory( - snapshot *sqlx.Tx, query string, args map[string]interface{}, checkpoint interface{}, - rowType reflect.Type, onRows func(idoRows interface{}) (checkpoint interface{}), +func sliceIdoHistory[Row any]( + snapshot *sqlx.Tx, query string, args map[string]interface{}, + checkpoint interface{}, onRows func([]Row) (checkpoint interface{}), ) { - vNewRows := reflect.New(reflect.SliceOf(rowType)) - rowsPtr := vNewRows.Interface() - vRows := vNewRows.Elem() - if args == nil { args = map[string]interface{}{} } @@ -142,21 +137,21 @@ func sliceIdoHistory( log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) } - if err := stmt.Select(rowsPtr, args); err != nil { + var rows []Row + if err := stmt.Select(&rows, args); err != nil { log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't perform query")) } _ = stmt.Close() - if vRows.Len() < 1 { + if len(rows) < 1 { break } - if checkpoint = onRows(vRows.Interface()); checkpoint == nil { + if checkpoint = onRows(rows); checkpoint == nil { break } - vRows.Set(vRows.Slice(0, 0)) args["checkpoint"] = checkpoint } } @@ -177,13 +172,9 @@ type historyType struct { cacheLimitQuery string // migrationQuery SELECTs source data for actual migration. migrationQuery string - // convertRowType specifies convertRows' idoRows. - convertRowType reflect.Type - // convertRows intention: see migrate(). - // TODO: make idoRows generic[T] one nice day - convertRows func(env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, - idoRows interface{}) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) + // migrate does the actual migration. + migrate func(c *Config, idb *icingadb.DB, envId []byte, + endpointId [sha1.Size]byte, idbTx *sync.Mutex, ht *historyType) // cache represents .sqlite3. cache *sqlx.DB @@ -241,24 +232,27 @@ var types = historyTypes{ }) }, migrationQuery: acknowledgementMigrationQuery, - convertRowType: reflect.TypeOf((*acknowledgementRow)(nil)).Elem(), - convertRows: convertAcknowledgementRows, + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { + migrateOneType(c, idb, envId, endpId, idbTx, ht, convertAcknowledgementRows) + }, }, { name: "comment", idoTable: "icinga_commenthistory", idoIdColumn: "commenthistory_id", migrationQuery: commentMigrationQuery, - convertRowType: reflect.TypeOf((*commentRow)(nil)).Elem(), - convertRows: convertCommentRows, + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { + migrateOneType(c, idb, envId, endpId, idbTx, ht, convertCommentRows) + }, }, { name: "downtime", idoTable: "icinga_downtimehistory", idoIdColumn: "downtimehistory_id", migrationQuery: downtimeMigrationQuery, - convertRowType: reflect.TypeOf((*downtimeRow)(nil)).Elem(), - convertRows: convertDowntimeRows, + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { + migrateOneType(c, idb, envId, endpId, idbTx, ht, convertDowntimeRows) + }, }, { name: "flapping", @@ -272,8 +266,9 @@ var types = historyTypes{ }) }, migrationQuery: flappingMigrationQuery, - convertRowType: reflect.TypeOf((*flappingRow)(nil)).Elem(), - convertRows: convertFlappingRows, + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { + migrateOneType(c, idb, envId, endpId, idbTx, ht, convertFlappingRows) + }, }, { name: "notification", @@ -287,8 +282,9 @@ var types = historyTypes{ }, cacheLimitQuery: "SELECT MAX(history_id) FROM previous_hard_state", migrationQuery: notificationMigrationQuery, - convertRowType: reflect.TypeOf((*notificationRow)(nil)).Elem(), - convertRows: convertNotificationRows, + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { + migrateOneType(c, idb, envId, endpId, idbTx, ht, convertNotificationRows) + }, }, { name: "state", @@ -300,7 +296,8 @@ var types = historyTypes{ }, cacheLimitQuery: "SELECT MAX(history_id) FROM previous_hard_state", migrationQuery: stateMigrationQuery, - convertRowType: reflect.TypeOf((*stateRow)(nil)).Elem(), - convertRows: convertStateRows, + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { + migrateOneType(c, idb, envId, endpId, idbTx, ht, convertStateRows) + }, }, } From 7dd911d9705ed250c010c38dd3a463a4735b44db Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 6 Apr 2022 16:35:50 +0200 Subject: [PATCH 052/103] cmd/ido2icingadb: better ETA --- cmd/ido2icingadb/cache.go | 7 ++--- cmd/ido2icingadb/main.go | 3 +- cmd/ido2icingadb/misc.go | 61 ++++++++++++++++++++++++++++++--------- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 3e06ec3d..d855069b 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -6,7 +6,6 @@ import ( "github.com/pkg/errors" "math" "strings" - "time" ) var eventTimeCacheSchema = []string{ @@ -49,7 +48,6 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { cacheGet(*tx, &checkpoint, "SELECT COUNT(*) cnt, MAX(history_id) max_id FROM end_start_time") ht.bar.SetCurrent(checkpoint.Cnt * 2) - inc := barIncrementer{ht.bar, time.Now()} // Stream source data... sliceIdoHistory( @@ -106,7 +104,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { checkpoint = idoRow.Id } - inc.inc(len(idoRows)) + ht.bar.IncrBy(len(idoRows)) return }, ) @@ -178,7 +176,6 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { } ht.bar.SetCurrent(previousHardStateCnt + nextIds.Cnt) - inc := barIncrementer{ht.bar, time.Now()} // We continue where we finished before. As we build the cache in reverse chronological order: // 1. If the history grows between two migration trials, we won't migrate the difference. Workarounds: @@ -250,7 +247,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { checkpoint = idoRow.Id } - inc.inc(len(idoRows)) + ht.bar.IncrBy(len(idoRows)) return }, ) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 55661889..cfee13dd 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -344,7 +344,6 @@ func migrateOneType[IdoRow any]( icingaDbUpdates := map[reflect.Type]string{} ht.bar.SetCurrent(ht.done) - inc := barIncrementer{ht.bar, time.Now()} // Stream IDO rows, ... sliceIdoHistory( @@ -417,7 +416,7 @@ func migrateOneType[IdoRow any]( log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } - inc.inc(len(idoRows)) + ht.bar.IncrBy(len(idoRows)) return lastIdoId }, ) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index a3d2f287..56cddc83 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -19,24 +19,53 @@ import ( "time" ) -// barIncrementer simplifies incrementing bar. -type barIncrementer struct { - // bar is the bar to increment. - bar *mpb.Bar - // start shall be the work start time. - start time.Time +// eta indicates the ETA for possibly slowly incrementing progresses possibly starting from >0%. +type eta struct { + decor.WC + + // startProgress is the first progress >0 seen by Decor. + startProgress int64 + // startTime tells when is startProgress from. + startTime time.Time + // lastProgress is the last progress >0 seen by Decor. + lastProgress int64 + // lastTime tells when is lastProgress from. + lastTime time.Time } -// inc increments bi.bar by i. -func (bi *barIncrementer) inc(i int) { - prev := bi.start - now := time.Now() - bi.start = now +// Decor implements the decor.Decorator interface. +func (e *eta) Decor(s decor.Statistics) string { + if s.Completed || s.Current < 1 { + return "" + } - bi.bar.IncrBy(i) - bi.bar.DecoratorEwmaUpdate(now.Sub(prev)) + if e.startProgress < 1 { + e.startProgress = s.Current + e.startTime = time.Now() + e.lastProgress = e.startProgress + e.lastTime = e.startTime + + return "" + } + + if s.Current == e.startProgress { + return "" + } + + if s.Current > e.lastProgress { + e.lastProgress = s.Current + e.lastTime = time.Now() + } + + timePerItem := float64(e.lastTime.Sub(e.startTime)) / float64(e.lastProgress-e.startProgress) + lastETA := time.Duration(float64(s.Total-s.Current) * timePerItem) + + return e.FormatMsg(((lastETA - time.Since(e.lastTime)) / time.Second * time.Second).String()) } +// Assert interface compliance. +var _ decor.Decorator = (*eta)(nil) + type IdoMigrationProgressUpserter struct { HistoryType string `json:"history_type"` } @@ -192,6 +221,10 @@ type historyType struct { // setupBar (re-)initializes ht.bar. func (ht *historyType) setupBar(progress *mpb.Progress) { + e := &eta{WC: decor.WC{W: 4}} + + e.Init() + ht.bar = progress.AddBar( ht.total, mpb.BarFillerClearOnComplete(), @@ -199,7 +232,7 @@ func (ht *historyType) setupBar(progress *mpb.Progress) { decor.Name(ht.name, decor.WC{W: len(ht.name) + 1, C: decor.DidentRight}), decor.Percentage(decor.WC{W: 5}), ), - mpb.AppendDecorators(decor.EwmaETA(decor.ET_STYLE_GO, 6000000, decor.WC{W: 4})), + mpb.AppendDecorators(e), ) } From 3b634ec49986768b5ad8fd5c698aefb47d120a6e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 12 May 2022 17:37:21 +0200 Subject: [PATCH 053/103] cmd/ido2icingadb: build fix --- cmd/ido2icingadb/convert.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index d36ca06a..b6b7163e 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -102,7 +102,7 @@ func convertAcknowledgementRows( typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) serviceId := calcServiceId(env, row.Name1, row.Name2) - setTime := float64(utils.UnixMilli(set.Time())) + setTime := float64(set.Time().UnixMilli()) acknowledgementHistoryId := hashAny([]interface{}{env, name, setTime}) if row.AcknowledgementType == 0 { // clear @@ -524,7 +524,7 @@ func convertFlappingRows( typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) serviceId := calcServiceId(env, row.Name1, row.Name2) - startTime := float64(utils.UnixMilli(start.Time())) + startTime := float64(start.Time().UnixMilli()) flappingHistoryId := hashAny([]interface{}{env, name, startTime}) if row.EventType == 1001 { // end @@ -705,7 +705,7 @@ func convertNotificationRows( } ts := convertTime(row.EndTime, row.EndTimeUsec) - tsMilli := float64(utils.UnixMilli(ts.Time())) + tsMilli := float64(ts.Time().UnixMilli()) notificationHistoryId := hashAny([]interface{}{env, name, ntEnum, tsMilli}) id := hashAny([]interface{}{env, "notification", name, ntEnum, tsMilli}) typ := objectTypes[row.ObjecttypeId] @@ -841,7 +841,7 @@ func convertStateRows( name := strings.Join([]string{row.Name1, row.Name2}, "!") ts := convertTime(row.StateTime, row.StateTimeUsec) - tsMilli := float64(utils.UnixMilli(ts.Time())) + tsMilli := float64(ts.Time().UnixMilli()) stateHistoryId := hashAny([]interface{}{env, name, tsMilli}) id := hashAny([]interface{}{env, "state_change", name, tsMilli}) typ := objectTypes[row.ObjecttypeId] From 67c7fe9257dd72907c2ef3585cd7581fe8b969b5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 12 May 2022 19:03:28 +0200 Subject: [PATCH 054/103] cmd/ido2icingadb: go:embed schemata for the sake of syntax highlighting. --- cmd/ido2icingadb/cache.go | 40 +++---------------- .../embed/event_time_cache_schema.sql | 13 ++++++ .../embed/ido_migration_progress_schema.sql | 6 +++ .../previous_hard_state_cache_schema.sql | 20 ++++++++++ cmd/ido2icingadb/main.go | 26 +++++------- cmd/ido2icingadb/misc.go | 2 +- 6 files changed, 56 insertions(+), 51 deletions(-) create mode 100644 cmd/ido2icingadb/embed/event_time_cache_schema.sql create mode 100644 cmd/ido2icingadb/embed/ido_migration_progress_schema.sql create mode 100644 cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index d855069b..25e38abe 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -2,26 +2,18 @@ package main import ( "database/sql" + _ "embed" "github.com/jmoiron/sqlx" "github.com/pkg/errors" "math" "strings" ) -var eventTimeCacheSchema = []string{ - // Icinga DB's flapping_history#start_time per flapping_end row (IDO's icinga_flappinghistory#flappinghistory_id). - `CREATE TABLE IF NOT EXISTS end_start_time ( - history_id INT PRIMARY KEY, - event_time INT NOT NULL, - event_time_usec INT NOT NULL -)`, - // Helper table, the last start_time per icinga_statehistory#object_id. - `CREATE TABLE IF NOT EXISTS last_start_time ( - object_id INT PRIMARY KEY, - event_time INT NOT NULL, - event_time_usec INT NOT NULL -)`, -} +//go:embed embed/event_time_cache_schema.sql +var eventTimeCacheSchema string + +//go:embed embed/previous_hard_state_cache_schema.sql +var previousHardStateCacheSchema string // buildEventTimeCache rationale: // @@ -116,26 +108,6 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { ht.bar.SetTotal(ht.bar.Current(), true) } -var previousHardStateCacheSchema = []string{ - // Icinga DB's state_history#previous_hard_state per IDO's icinga_statehistory#statehistory_id. - `CREATE TABLE IF NOT EXISTS previous_hard_state ( - history_id INT PRIMARY KEY, - previous_hard_state INT NOT NULL -)`, - // Helper table, the current last_hard_state per icinga_statehistory#object_id. - `CREATE TABLE IF NOT EXISTS next_hard_state ( - object_id INT PRIMARY KEY, - next_hard_state INT NOT NULL -)`, - // Helper table for stashing icinga_statehistory#statehistory_id until last_hard_state changes. - `CREATE TABLE IF NOT EXISTS next_ids ( - object_id INT NOT NULL, - history_id INT NOT NULL -)`, - "CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids(object_id)", - "CREATE INDEX IF NOT EXISTS next_ids_history_id ON next_ids(history_id)", -} - // buildPreviousHardStateCache rationale: // // Icinga DB's state_history#previous_hard_state would need a subquery. diff --git a/cmd/ido2icingadb/embed/event_time_cache_schema.sql b/cmd/ido2icingadb/embed/event_time_cache_schema.sql new file mode 100644 index 00000000..2818b5b0 --- /dev/null +++ b/cmd/ido2icingadb/embed/event_time_cache_schema.sql @@ -0,0 +1,13 @@ +-- Icinga DB's flapping_history#start_time per flapping_end row (IDO's icinga_flappinghistory#flappinghistory_id). +CREATE TABLE IF NOT EXISTS end_start_time ( + history_id INT PRIMARY KEY, + event_time INT NOT NULL, + event_time_usec INT NOT NULL +); + +-- Helper table, the last start_time per icinga_statehistory#object_id. +CREATE TABLE IF NOT EXISTS last_start_time ( + object_id INT PRIMARY KEY, + event_time INT NOT NULL, + event_time_usec INT NOT NULL +); diff --git a/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql b/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql new file mode 100644 index 00000000..aa682484 --- /dev/null +++ b/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS ido_migration_progress ( + history_type VARCHAR(63) NOT NULL, + last_ido_id BIGINT NOT NULL, + + CONSTRAINT pk_ido_migration_progress PRIMARY KEY (history_type) +); diff --git a/cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql b/cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql new file mode 100644 index 00000000..eb0a62e3 --- /dev/null +++ b/cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql @@ -0,0 +1,20 @@ +-- Icinga DB's state_history#previous_hard_state per IDO's icinga_statehistory#statehistory_id. +CREATE TABLE IF NOT EXISTS previous_hard_state ( + history_id INT PRIMARY KEY, + previous_hard_state INT NOT NULL +); + +-- Helper table, the current last_hard_state per icinga_statehistory#object_id. +CREATE TABLE IF NOT EXISTS next_hard_state ( + object_id INT PRIMARY KEY, + next_hard_state INT NOT NULL +); + +-- Helper table for stashing icinga_statehistory#statehistory_id until last_hard_state changes. +CREATE TABLE IF NOT EXISTS next_ids ( + object_id INT NOT NULL, + history_id INT NOT NULL +); + +CREATE INDEX IF NOT EXISTS next_ids_object_id ON next_ids (object_id); +CREATE INDEX IF NOT EXISTS next_ids_history_id ON next_ids (history_id); diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index cfee13dd..ad1f1997 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha1" "database/sql" + _ "embed" "encoding/hex" "fmt" "github.com/goccy/go-yaml" @@ -121,7 +122,7 @@ func mkCache(f *Flags, mapper *reflectx.Mapper) { } types.forEach(func(ht *historyType) { - if ht.cacheSchema == nil { + if ht.cacheSchema == "" { return } @@ -135,11 +136,9 @@ func mkCache(f *Flags, mapper *reflectx.Mapper) { ht.cache.Mapper = mapper - for _, ddl := range ht.cacheSchema { - if _, err := ht.cache.Exec(ddl); err != nil { - log.With("file", file, "ddl", ddl). - Fatalf("%+v", errors.Wrap(err, "can't import schema into SQLite database")) - } + if _, err := ht.cache.Exec(ht.cacheSchema); err != nil { + log.With("file", file, "ddl", ht.cacheSchema). + Fatalf("%+v", errors.Wrap(err, "can't import schema into SQLite database")) } }) } @@ -190,19 +189,14 @@ func startIdoTx(ido *icingadb.DB) { }) } +//go:embed embed/ido_migration_progress_schema.sql +var idoMigrationProgressSchema string + // computeProgress initializes types[*].lastId, types[*].total and types[*].done. // (On non-recoverable errors the whole program exits.) func computeProgress(idb *icingadb.DB) { - { - _, err := idb.Exec(`CREATE TABLE IF NOT EXISTS ido_migration_progress ( - history_type VARCHAR(63) NOT NULL, - last_ido_id BIGINT NOT NULL, - - CONSTRAINT pk_ido_migration_progress PRIMARY KEY (history_type) -)`) - if err != nil { - log.Fatalf("%+v", errors.Wrap(err, "can't create table ido_migration_progress")) - } + if _, err := idb.Exec(idoMigrationProgressSchema); err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't create table ido_migration_progress")) } stmt, _ := idb.BuildUpsertStmt(&IdoMigrationProgress{}) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 56cddc83..258302a2 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -194,7 +194,7 @@ type historyType struct { // idoIdColumn specifies idoTable's primary key. idoIdColumn string // cacheSchema specifies .sqlite3's structure. - cacheSchema []string + cacheSchema string // cacheFiller fills cache from snapshot. cacheFiller func(*historyType) // cacheLimitQuery rationale: see migrate(). From 5470c297b1ed42499e53cadfc0dc5190ab4a0616 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 19 May 2022 11:46:49 +0200 Subject: [PATCH 055/103] cmd/ido2icingadb: go:embed large queries for the sake of syntax highlighting. --- cmd/ido2icingadb/convert.go | 83 ++++--------------- cmd/ido2icingadb/embed/ack_query.sql | 8 ++ cmd/ido2icingadb/embed/comment_query.sql | 10 +++ cmd/ido2icingadb/embed/downtime_query.sql | 13 +++ cmd/ido2icingadb/embed/flapping_query.sql | 8 ++ cmd/ido2icingadb/embed/notification_query.sql | 8 ++ cmd/ido2icingadb/embed/state_query.sql | 8 ++ 7 files changed, 73 insertions(+), 65 deletions(-) create mode 100644 cmd/ido2icingadb/embed/ack_query.sql create mode 100644 cmd/ido2icingadb/embed/comment_query.sql create mode 100644 cmd/ido2icingadb/embed/downtime_query.sql create mode 100644 cmd/ido2icingadb/embed/flapping_query.sql create mode 100644 cmd/ido2icingadb/embed/notification_query.sql create mode 100644 cmd/ido2icingadb/embed/state_query.sql diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index b6b7163e..5e2be7e3 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -2,6 +2,7 @@ package main import ( "database/sql" + _ "embed" "github.com/icinga/icingadb/pkg/contracts" v1 "github.com/icinga/icingadb/pkg/icingadb/v1" "github.com/icinga/icingadb/pkg/icingadb/v1/history" @@ -14,15 +15,23 @@ import ( "time" ) -const acknowledgementMigrationQuery = "SELECT ah.acknowledgement_id, UNIX_TIMESTAMP(ah.entry_time) entry_time, " + - "ah.entry_time_usec, ah.acknowledgement_type, ah.author_name, ah.comment_data, ah.is_sticky, " + - "ah.persistent_comment, UNIX_TIMESTAMP(ah.end_time) end_time, o.objecttype_id, o.name1, " + - "COALESCE(o.name2, '') name2 " + - "FROM icinga_acknowledgements ah USE INDEX (PRIMARY) " + - "INNER JOIN icinga_objects o ON o.object_id=ah.object_id " + - "WHERE ah.acknowledgement_id > :checkpoint " + // where we were interrupted - "ORDER BY ah.acknowledgement_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT :bulk" +//go:embed embed/ack_query.sql +var acknowledgementMigrationQuery string + +//go:embed embed/comment_query.sql +var commentMigrationQuery string + +//go:embed embed/downtime_query.sql +var downtimeMigrationQuery string + +//go:embed embed/flapping_query.sql +var flappingMigrationQuery string + +//go:embed embed/notification_query.sql +var notificationMigrationQuery string + +//go:embed embed/state_query.sql +var stateMigrationQuery string // AckClear updates an already migrated ack event with the clear event info. type AckClear struct { @@ -178,17 +187,6 @@ func convertAcknowledgementRows( return } -const commentMigrationQuery = "SELECT ch.commenthistory_id, UNIX_TIMESTAMP(ch.entry_time) entry_time, " + - "ch.entry_time_usec, ch.entry_type, ch.author_name, ch.comment_data, ch.is_persistent, " + - "COALESCE(UNIX_TIMESTAMP(ch.expiration_time), 0) expiration_time, " + - "COALESCE(UNIX_TIMESTAMP(ch.deletion_time), 0) deletion_time, ch.deletion_time_usec, ch.name, " + - "o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 " + - "FROM icinga_commenthistory ch USE INDEX (PRIMARY) " + - "INNER JOIN icinga_objects o ON o.object_id=ch.object_id " + - "WHERE ch.commenthistory_id > :checkpoint " + // where we were interrupted - "ORDER BY ch.commenthistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT :bulk" - type commentRow = struct { CommenthistoryId uint64 EntryTime int64 @@ -288,21 +286,6 @@ func convertCommentRows( return } -const downtimeMigrationQuery = "SELECT dh.downtimehistory_id, UNIX_TIMESTAMP(dh.entry_time) entry_time, " + - "dh.author_name, dh.comment_data, dh.is_fixed, dh.duration, " + - "UNIX_TIMESTAMP(dh.scheduled_start_time) scheduled_start_time, " + - "COALESCE(UNIX_TIMESTAMP(dh.scheduled_end_time), 0) scheduled_end_time, " + - "COALESCE(UNIX_TIMESTAMP(dh.actual_start_time), 0) actual_start_time, dh.actual_start_time_usec, " + - "COALESCE(UNIX_TIMESTAMP(dh.actual_end_time), 0) actual_end_time, dh.actual_end_time_usec, dh.was_cancelled, " + - "COALESCE(UNIX_TIMESTAMP(dh.trigger_time), 0) trigger_time, dh.name, o.objecttype_id, o.name1, " + - "COALESCE(o.name2, '') name2, COALESCE(sd.name, '') triggered_by " + - "FROM icinga_downtimehistory dh USE INDEX (PRIMARY) " + - "INNER JOIN icinga_objects o ON o.object_id=dh.object_id " + - "LEFT JOIN icinga_scheduleddowntime sd ON sd.scheduleddowntime_id=dh.triggered_by_id " + - "WHERE dh.downtimehistory_id > :checkpoint " + // where we were interrupted - "ORDER BY dh.downtimehistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT :bulk" - type downtimeRow = struct { DowntimehistoryId uint64 EntryTime int64 @@ -439,15 +422,6 @@ func convertDowntimeRows( return } -const flappingMigrationQuery = "SELECT fh.flappinghistory_id, UNIX_TIMESTAMP(fh.event_time) event_time, " + - "fh.event_time_usec, fh.event_type, fh.percent_state_change, fh.low_threshold, " + - "fh.high_threshold, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 " + - "FROM icinga_flappinghistory fh USE INDEX (PRIMARY) " + - "INNER JOIN icinga_objects o ON o.object_id=fh.object_id " + - "WHERE fh.flappinghistory_id > :checkpoint " + // where we were interrupted - "ORDER BY fh.flappinghistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT :bulk" - // FlappingEnd updates an already migrated start event with the end event info. type FlappingEnd struct { Id icingadbTypes.Binary @@ -598,16 +572,6 @@ func convertFlappingRows( return } -const notificationMigrationQuery = "SELECT n.notification_id, n.notification_reason, " + - "UNIX_TIMESTAMP(n.end_time) end_time, n.end_time_usec, n.state, COALESCE(n.output, '') output, " + - "n.long_output, n.contacts_notified, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 " + - "FROM icinga_notifications n USE INDEX (PRIMARY) " + - "INNER JOIN icinga_objects o ON o.object_id=n.object_id " + - "WHERE n.notification_id <= :cache_limit AND " + - "n.notification_id > :checkpoint " + // where we were interrupted - "ORDER BY n.notification_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT :bulk" - // notificationTypes maps IDO values to Icinga DB ones. var notificationTypes = map[uint8]icingadbTypes.NotificationType{5: 1, 6: 2, 7: 4, 8: 8, 1: 16, 2: 128, 3: 256} @@ -781,17 +745,6 @@ func convertNotificationRows( return } -const stateMigrationQuery = "SELECT sh.statehistory_id, UNIX_TIMESTAMP(sh.state_time) state_time, " + - "sh.state_time_usec, sh.state, sh.state_type, sh.current_check_attempt, " + - "sh.max_check_attempts, sh.last_state, sh.last_hard_state, sh.output, sh.long_output, " + - "sh.check_source, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 " + - "FROM icinga_statehistory sh USE INDEX (PRIMARY) " + - "INNER JOIN icinga_objects o ON o.object_id=sh.object_id " + - "WHERE sh.statehistory_id <= :cache_limit AND " + - "sh.statehistory_id > :checkpoint " + // where we were interrupted - "ORDER BY sh.statehistory_id " + // allows computeProgress() not to check all IDO rows for whether migrated - "LIMIT :bulk" - type stateRow = struct { StatehistoryId uint64 StateTime int64 diff --git a/cmd/ido2icingadb/embed/ack_query.sql b/cmd/ido2icingadb/embed/ack_query.sql new file mode 100644 index 00000000..998e450f --- /dev/null +++ b/cmd/ido2icingadb/embed/ack_query.sql @@ -0,0 +1,8 @@ +SELECT ah.acknowledgement_id, UNIX_TIMESTAMP(ah.entry_time) entry_time, ah.entry_time_usec, + ah.acknowledgement_type, ah.author_name, ah.comment_data, ah.is_sticky, ah.persistent_comment, + UNIX_TIMESTAMP(ah.end_time) end_time, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 +FROM icinga_acknowledgements ah USE INDEX (PRIMARY) +INNER JOIN icinga_objects o ON o.object_id=ah.object_id +WHERE ah.acknowledgement_id > :checkpoint -- where we were interrupted +ORDER BY ah.acknowledgement_id -- allows computeProgress() not to check all IDO rows for whether migrated +LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/comment_query.sql b/cmd/ido2icingadb/embed/comment_query.sql new file mode 100644 index 00000000..ea71087c --- /dev/null +++ b/cmd/ido2icingadb/embed/comment_query.sql @@ -0,0 +1,10 @@ +SELECT ch.commenthistory_id, UNIX_TIMESTAMP(ch.entry_time) entry_time, + ch.entry_time_usec, ch.entry_type, ch.author_name, ch.comment_data, ch.is_persistent, + COALESCE(UNIX_TIMESTAMP(ch.expiration_time), 0) expiration_time, + COALESCE(UNIX_TIMESTAMP(ch.deletion_time), 0) deletion_time, + ch.deletion_time_usec, ch.name, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 +FROM icinga_commenthistory ch USE INDEX (PRIMARY) +INNER JOIN icinga_objects o ON o.object_id=ch.object_id +WHERE ch.commenthistory_id > :checkpoint -- where we were interrupted +ORDER BY ch.commenthistory_id -- allows computeProgress() not to check all IDO rows for whether migrated +LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/downtime_query.sql b/cmd/ido2icingadb/embed/downtime_query.sql new file mode 100644 index 00000000..993db369 --- /dev/null +++ b/cmd/ido2icingadb/embed/downtime_query.sql @@ -0,0 +1,13 @@ +SELECT dh.downtimehistory_id, UNIX_TIMESTAMP(dh.entry_time) entry_time, dh.author_name, dh.comment_data, + dh.is_fixed, dh.duration, UNIX_TIMESTAMP(dh.scheduled_start_time) scheduled_start_time, + COALESCE(UNIX_TIMESTAMP(dh.scheduled_end_time), 0) scheduled_end_time, + COALESCE(UNIX_TIMESTAMP(dh.actual_start_time), 0) actual_start_time, dh.actual_start_time_usec, + COALESCE(UNIX_TIMESTAMP(dh.actual_end_time), 0) actual_end_time, dh.actual_end_time_usec, dh.was_cancelled, + COALESCE(UNIX_TIMESTAMP(dh.trigger_time), 0) trigger_time, dh.name, o.objecttype_id, + o.name1, COALESCE(o.name2, '') name2, COALESCE(sd.name, '') triggered_by +FROM icinga_downtimehistory dh USE INDEX (PRIMARY) +INNER JOIN icinga_objects o ON o.object_id=dh.object_id +LEFT JOIN icinga_scheduleddowntime sd ON sd.scheduleddowntime_id=dh.triggered_by_id +WHERE dh.downtimehistory_id > :checkpoint -- where we were interrupted +ORDER BY dh.downtimehistory_id -- allows computeProgress() not to check all IDO rows for whether migrated +LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/flapping_query.sql b/cmd/ido2icingadb/embed/flapping_query.sql new file mode 100644 index 00000000..6abed6eb --- /dev/null +++ b/cmd/ido2icingadb/embed/flapping_query.sql @@ -0,0 +1,8 @@ +SELECT fh.flappinghistory_id, UNIX_TIMESTAMP(fh.event_time) event_time, + fh.event_time_usec, fh.event_type, fh.percent_state_change, fh.low_threshold, + fh.high_threshold, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 +FROM icinga_flappinghistory fh USE INDEX (PRIMARY) +INNER JOIN icinga_objects o ON o.object_id=fh.object_id +WHERE fh.flappinghistory_id > :checkpoint -- where we were interrupted +ORDER BY fh.flappinghistory_id -- allows computeProgress() not to check all IDO rows for whether migrated +LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/notification_query.sql b/cmd/ido2icingadb/embed/notification_query.sql new file mode 100644 index 00000000..2216db3d --- /dev/null +++ b/cmd/ido2icingadb/embed/notification_query.sql @@ -0,0 +1,8 @@ +SELECT n.notification_id, n.notification_reason, UNIX_TIMESTAMP(n.end_time) end_time, + n.end_time_usec, n.state, COALESCE(n.output, '') output, n.long_output, + n.contacts_notified, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 +FROM icinga_notifications n USE INDEX (PRIMARY) +INNER JOIN icinga_objects o ON o.object_id=n.object_id +WHERE n.notification_id <= :cache_limit AND n.notification_id > :checkpoint -- where we were interrupted +ORDER BY n.notification_id -- allows computeProgress() not to check all IDO rows for whether migrated +LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/state_query.sql b/cmd/ido2icingadb/embed/state_query.sql new file mode 100644 index 00000000..d53b0f91 --- /dev/null +++ b/cmd/ido2icingadb/embed/state_query.sql @@ -0,0 +1,8 @@ +SELECT sh.statehistory_id, UNIX_TIMESTAMP(sh.state_time) state_time, sh.state_time_usec, sh.state, + sh.state_type, sh.current_check_attempt, sh.max_check_attempts, sh.last_state, sh.last_hard_state, + sh.output, sh.long_output, sh.check_source, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 +FROM icinga_statehistory sh USE INDEX (PRIMARY) +INNER JOIN icinga_objects o ON o.object_id=sh.object_id +WHERE sh.statehistory_id <= :cache_limit AND sh.statehistory_id > :checkpoint -- where we were interrupted +ORDER BY sh.statehistory_id -- allows computeProgress() not to check all IDO rows for whether migrated +LIMIT :bulk From 7c6f9ddbbd315ca3c31df05904ed291b5326364d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 19 May 2022 17:11:45 +0200 Subject: [PATCH 056/103] cmd/ido2icingadb: improve comments --- cmd/ido2icingadb/embed/ack_query.sql | 2 +- cmd/ido2icingadb/embed/comment_query.sql | 2 +- cmd/ido2icingadb/embed/downtime_query.sql | 2 +- cmd/ido2icingadb/embed/flapping_query.sql | 2 +- cmd/ido2icingadb/embed/notification_query.sql | 2 +- cmd/ido2icingadb/embed/state_query.sql | 2 +- cmd/ido2icingadb/main.go | 2 ++ 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/ido2icingadb/embed/ack_query.sql b/cmd/ido2icingadb/embed/ack_query.sql index 998e450f..4ba2a441 100644 --- a/cmd/ido2icingadb/embed/ack_query.sql +++ b/cmd/ido2icingadb/embed/ack_query.sql @@ -4,5 +4,5 @@ SELECT ah.acknowledgement_id, UNIX_TIMESTAMP(ah.entry_time) entry_time, ah.entry FROM icinga_acknowledgements ah USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=ah.object_id WHERE ah.acknowledgement_id > :checkpoint -- where we were interrupted -ORDER BY ah.acknowledgement_id -- allows computeProgress() not to check all IDO rows for whether migrated +ORDER BY ah.acknowledgement_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/comment_query.sql b/cmd/ido2icingadb/embed/comment_query.sql index ea71087c..5165af3d 100644 --- a/cmd/ido2icingadb/embed/comment_query.sql +++ b/cmd/ido2icingadb/embed/comment_query.sql @@ -6,5 +6,5 @@ SELECT ch.commenthistory_id, UNIX_TIMESTAMP(ch.entry_time) entry_time, FROM icinga_commenthistory ch USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=ch.object_id WHERE ch.commenthistory_id > :checkpoint -- where we were interrupted -ORDER BY ch.commenthistory_id -- allows computeProgress() not to check all IDO rows for whether migrated +ORDER BY ch.commenthistory_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/downtime_query.sql b/cmd/ido2icingadb/embed/downtime_query.sql index 993db369..34b9f0ad 100644 --- a/cmd/ido2icingadb/embed/downtime_query.sql +++ b/cmd/ido2icingadb/embed/downtime_query.sql @@ -9,5 +9,5 @@ FROM icinga_downtimehistory dh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=dh.object_id LEFT JOIN icinga_scheduleddowntime sd ON sd.scheduleddowntime_id=dh.triggered_by_id WHERE dh.downtimehistory_id > :checkpoint -- where we were interrupted -ORDER BY dh.downtimehistory_id -- allows computeProgress() not to check all IDO rows for whether migrated +ORDER BY dh.downtimehistory_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/flapping_query.sql b/cmd/ido2icingadb/embed/flapping_query.sql index 6abed6eb..f6f3055d 100644 --- a/cmd/ido2icingadb/embed/flapping_query.sql +++ b/cmd/ido2icingadb/embed/flapping_query.sql @@ -4,5 +4,5 @@ SELECT fh.flappinghistory_id, UNIX_TIMESTAMP(fh.event_time) event_time, FROM icinga_flappinghistory fh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=fh.object_id WHERE fh.flappinghistory_id > :checkpoint -- where we were interrupted -ORDER BY fh.flappinghistory_id -- allows computeProgress() not to check all IDO rows for whether migrated +ORDER BY fh.flappinghistory_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/notification_query.sql b/cmd/ido2icingadb/embed/notification_query.sql index 2216db3d..63e1f720 100644 --- a/cmd/ido2icingadb/embed/notification_query.sql +++ b/cmd/ido2icingadb/embed/notification_query.sql @@ -4,5 +4,5 @@ SELECT n.notification_id, n.notification_reason, UNIX_TIMESTAMP(n.end_time) end_ FROM icinga_notifications n USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=n.object_id WHERE n.notification_id <= :cache_limit AND n.notification_id > :checkpoint -- where we were interrupted -ORDER BY n.notification_id -- allows computeProgress() not to check all IDO rows for whether migrated +ORDER BY n.notification_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/state_query.sql b/cmd/ido2icingadb/embed/state_query.sql index d53b0f91..00a92a8d 100644 --- a/cmd/ido2icingadb/embed/state_query.sql +++ b/cmd/ido2icingadb/embed/state_query.sql @@ -4,5 +4,5 @@ SELECT sh.statehistory_id, UNIX_TIMESTAMP(sh.state_time) state_time, sh.state_ti FROM icinga_statehistory sh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=sh.object_id WHERE sh.statehistory_id <= :cache_limit AND sh.statehistory_id > :checkpoint -- where we were interrupted -ORDER BY sh.statehistory_id -- allows computeProgress() not to check all IDO rows for whether migrated +ORDER BY sh.statehistory_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index ad1f1997..5330bbcc 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -351,6 +351,8 @@ func migrateOneType[IdoRow any]( // ... and insert them: if idb.DriverName() == driver.MySQL { + // Avoid MySQL error 1205 (Lock wait timeout exceeded; try restarting transaction) + // due to concurrent transactions upsert the same table (history). idbTx.Lock() defer idbTx.Unlock() } From 041d4a0a74c02de51c634e184518f949c0aaa7d1 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 19 May 2022 17:26:33 +0200 Subject: [PATCH 057/103] cmd/ido2icingadb: avoid &(*ht)[i] --- cmd/ido2icingadb/main.go | 10 +++++----- cmd/ido2icingadb/misc.go | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 5330bbcc..f647b0bd 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -252,9 +252,9 @@ func computeProgress(idb *icingadb.DB) { // fillCache fills /.sqlite3 (actually types[*].cacheFiller does). func fillCache() { progress := mpb.New() - for i := range types { - if types[i].cacheFiller != nil { - types[i].setupBar(progress) + for _, ht := range types { + if ht.cacheFiller != nil { + ht.setupBar(progress) } } @@ -273,8 +273,8 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { idbTx := &sync.Mutex{} progress := mpb.New() - for i := range types { - types[i].setupBar(progress) + for _, ht := range types { + ht.setupBar(progress) } types.forEach(func(ht *historyType) { diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 258302a2..a69db2ae 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -236,15 +236,15 @@ func (ht *historyType) setupBar(progress *mpb.Progress) { ) } -type historyTypes [6]historyType +type historyTypes []*historyType -// forEach performs f per *ht in parallel. -func (ht *historyTypes) forEach(f func(*historyType)) { +// forEach performs f per hts in parallel. +func (hts historyTypes) forEach(f func(*historyType)) { eg, _ := errgroup.WithContext(context.Background()) - for i := range *ht { - i := i + for _, ht := range hts { + ht := ht eg.Go(func() error { - f(&(*ht)[i]) + f(ht) return nil }) } From 1a1f19163fecde1c8d9af6004f970893df27aaa5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 19 May 2022 19:01:04 +0200 Subject: [PATCH 058/103] cmd/ido2icingadb: write SLA --- cmd/ido2icingadb/convert.go | 53 +++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index 5e2be7e3..f16ba31d 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -312,7 +312,7 @@ func convertDowntimeRows( env string, envId, endpointId icingadbTypes.Binary, _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []downtimeRow, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { - var downtimeHistory, allHistory []interface{} + var downtimeHistory, allHistory, sla []interface{} for _, row := range idoRows { id := calcObjectId(env, row.Name) @@ -324,7 +324,7 @@ func convertDowntimeRows( triggerTime := convertTime(row.TriggerTime, 0) actualStart := convertTime(row.ActualStartTime, row.ActualStartTimeUsec) actualEnd := convertTime(row.ActualEndTime, row.ActualEndTimeUsec) - var startTime, endTime, cancelTime icingadbTypes.UnixMilli + var startTime, endTime, cancelTime, slaEndTime icingadbTypes.UnixMilli if scheduledEnd.Time().IsZero() { scheduledEnd = icingadbTypes.UnixMilli(scheduledStart.Time().Add(time.Duration(row.Duration) * time.Second)) @@ -346,8 +346,10 @@ func convertDowntimeRows( triggerTime = startTime } + slaEndTime = endTime if row.WasCancelled != 0 { cancelTime = actualEnd + slaEndTime = cancelTime } downtimeHistory = append(downtimeHistory, &history.DowntimeHistory{ @@ -415,10 +417,28 @@ func convertDowntimeRows( allHistory = append(allHistory, h2) } + s := &history.SlaHistoryDowntime{ + DowntimeHistoryEntity: history.DowntimeHistoryEntity{DowntimeId: id}, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + DowntimeStart: startTime, + HasBeenCancelled: icingadbTypes.Bool{Bool: row.WasCancelled != 0, Valid: true}, + CancelTime: cancelTime, + EndTime: slaEndTime, + } + + s.DowntimeEnd.History = s + sla = append(sla, s) + checkpoint = row.DowntimehistoryId } - icingaDbInserts = [][]interface{}{downtimeHistory, allHistory} + icingaDbInserts = [][]interface{}{downtimeHistory, allHistory, sla} return } @@ -785,7 +805,7 @@ func convertStateRows( cachedById[c.HistoryId] = c.PreviousHardState } - var stateHistory, allHistory []interface{} + var stateHistory, allHistory, sla []interface{} for _, row := range idoRows { previousHardState, ok := cachedById[row.StatehistoryId] if !ok { @@ -841,9 +861,32 @@ func convertStateRows( EventTime: ts, }) + if icingadbTypes.StateType(row.StateType) == icingadbTypes.StateHard { + // only hard state changes are relevant for SLA history, discard all others + + sla = append(sla, &history.SlaHistoryState{ + HistoryTableEntity: history.HistoryTableEntity{ + EntityWithoutChecksum: v1.EntityWithoutChecksum{ + IdMeta: v1.IdMeta{Id: stateHistoryId}, + }, + }, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + EventTime: ts, + StateType: icingadbTypes.StateType(row.StateType), + HardState: row.LastHardState, + PreviousHardState: previousHardState, + }) + } + checkpoint = row.StatehistoryId } - icingaDbInserts = [][]interface{}{stateHistory, allHistory} + icingaDbInserts = [][]interface{}{stateHistory, allHistory, sla} return } From 3cc8641276daa1e62c7f348fdaa65181dc0ddafd Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 13 Jun 2022 15:23:48 +0200 Subject: [PATCH 059/103] cmd/ido2icingadb: let SQLite VACUUM automatically --- cmd/ido2icingadb/cache.go | 4 ---- cmd/ido2icingadb/embed/event_time_cache_schema.sql | 2 ++ cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 25e38abe..16a82d43 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -283,10 +283,6 @@ func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onDeleted func(sql.Resul if err := tx.Commit(); err != nil { logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } - - if totalAffectedByDeletes > 0 { - cacheExec(cache, true, "VACUUM") - } } // cacheGet does cache.Get(dest, query, args...). (On non-recoverable errors the whole program exits.) diff --git a/cmd/ido2icingadb/embed/event_time_cache_schema.sql b/cmd/ido2icingadb/embed/event_time_cache_schema.sql index 2818b5b0..5940754b 100644 --- a/cmd/ido2icingadb/embed/event_time_cache_schema.sql +++ b/cmd/ido2icingadb/embed/event_time_cache_schema.sql @@ -1,3 +1,5 @@ +PRAGMA main.auto_vacuum = 1; + -- Icinga DB's flapping_history#start_time per flapping_end row (IDO's icinga_flappinghistory#flappinghistory_id). CREATE TABLE IF NOT EXISTS end_start_time ( history_id INT PRIMARY KEY, diff --git a/cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql b/cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql index eb0a62e3..315f22dc 100644 --- a/cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql +++ b/cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql @@ -1,3 +1,5 @@ +PRAGMA main.auto_vacuum = 1; + -- Icinga DB's state_history#previous_hard_state per IDO's icinga_statehistory#statehistory_id. CREATE TABLE IF NOT EXISTS previous_hard_state ( history_id INT PRIMARY KEY, From ad0b6cad91258129454fcc8ac17b5331988c8072 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 13 Jun 2022 15:44:43 +0200 Subject: [PATCH 060/103] cmd/ido2icingadb: remove obsolete code --- cmd/ido2icingadb/cache.go | 74 +++++++++++++-------------------------- 1 file changed, 25 insertions(+), 49 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 16a82d43..39546211 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -32,7 +32,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { ObjectId uint64 } - chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func()) { + chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onNewUncommittedDml func()) { var checkpoint struct { Cnt int64 MaxId sql.NullInt64 @@ -67,27 +67,22 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { if len(lst) > 0 { // ... save the start event time for the actual migration: cacheExec( - *tx, false, + *tx, "INSERT INTO end_start_time(history_id, event_time, event_time_usec) VALUES (?, ?, ?)", idoRow.Id, lst[0].EventTime, lst[0].EventTimeUsec, ) // This previously queried info isn't needed anymore. - onDeleted(cacheExec( - *tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId, - )) + cacheExec(*tx, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId) } } else { // Ack/flapping start event directly after another start event (per checkable). // The old one won't have (but the new one will) an end event (which will need its time). - onDeleted(cacheExec( - *tx, false, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId, - )) + cacheExec(*tx, "DELETE FROM last_start_time WHERE object_id=?", idoRow.ObjectId) // An ack/flapping start event. The following end event (per checkable) will need its time. cacheExec( - *tx, false, - "INSERT INTO last_start_time(object_id, event_time, event_time_usec) VALUES (?, ?, ?)", + *tx, "INSERT INTO last_start_time(object_id, event_time, event_time_usec) VALUES (?, ?, ?)", idoRow.ObjectId, idoRow.EventTime, idoRow.EventTimeUsec, ) } @@ -102,7 +97,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { ) // This never queried info isn't needed anymore. - onDeleted(cacheExec(*tx, false, "DELETE FROM last_start_time")) + cacheExec(*tx, "DELETE FROM last_start_time") }) ht.bar.SetTotal(ht.bar.Current(), true) @@ -123,7 +118,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { LastHardState uint8 } - chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func()) { + chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onNewUncommittedDml func()) { var nextIds struct { Cnt int64 MinId sql.NullInt64 @@ -173,44 +168,45 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { if len(nhs) < 1 { // we just started (per checkable) // At the moment (we're "travelling back in time") that's the checkable's hard state: cacheExec( - *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", + *tx, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", idoRow.ObjectId, idoRow.LastHardState, ) // But for the current time point the previous hard state isn't known, yet: cacheExec( - *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", + *tx, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", idoRow.Id, idoRow.ObjectId, ) } else if idoRow.LastHardState == nhs[0].NextHardState { // The hard state didn't change yet (per checkable), // so this time point also awaits the previous hard state. cacheExec( - *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", + *tx, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", idoRow.Id, idoRow.ObjectId, ) } else { // the hard state changed (per checkable) // That past hard state is now available for the processed future time points: cacheExec( - *tx, false, "INSERT INTO previous_hard_state(history_id, previous_hard_state) "+ + *tx, + "INSERT INTO previous_hard_state(history_id, previous_hard_state) "+ "SELECT history_id, ? FROM next_ids WHERE object_id=?", idoRow.LastHardState, idoRow.ObjectId, ) // Now they have what they wanted: - onDeleted(cacheExec(*tx, false, "DELETE FROM next_hard_state WHERE object_id=?", idoRow.ObjectId)) - onDeleted(cacheExec(*tx, false, "DELETE FROM next_ids WHERE object_id=?", idoRow.ObjectId)) + cacheExec(*tx, "DELETE FROM next_hard_state WHERE object_id=?", idoRow.ObjectId) + cacheExec(*tx, "DELETE FROM next_ids WHERE object_id=?", idoRow.ObjectId) // That's done. // Now do the same thing as in the "we just started" case above, for the same reason: cacheExec( - *tx, false, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", + *tx, "INSERT INTO next_hard_state(object_id, next_hard_state) VALUES (?, ?)", idoRow.ObjectId, idoRow.LastHardState, ) cacheExec( - *tx, false, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", + *tx, "INSERT INTO next_ids(history_id, object_id) VALUES (?, ?)", idoRow.Id, idoRow.ObjectId, ) } @@ -226,25 +222,22 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { // No past hard state is available for the processed future time points, assuming pending: cacheExec( - *tx, false, "INSERT INTO previous_hard_state(history_id, previous_hard_state) "+ - "SELECT history_id, 99 FROM next_ids", + *tx, "INSERT INTO previous_hard_state(history_id, previous_hard_state) SELECT history_id, 99 FROM next_ids", ) // Now they should have what they wanted: - onDeleted(cacheExec(*tx, false, "DELETE FROM next_hard_state")) - onDeleted(cacheExec(*tx, false, "DELETE FROM next_ids")) + cacheExec(*tx, "DELETE FROM next_hard_state") + cacheExec(*tx, "DELETE FROM next_ids") }) ht.bar.SetTotal(ht.bar.Current(), true) } // chunkCacheTx rationale: during do operate on cache via *tx. On every completed operation call onNewUncommittedDml() -// which periodically commits *tx and starts a new tx. (That's why tx is a **, not just a *.) On every DELETE -// call onDeleted() which will cause a VACUUM after do if any rows were affected to save some space. +// which periodically commits *tx and starts a new tx. (That's why tx is a **, not just a *.) // (On non-recoverable errors the whole program exits.) -func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onDeleted func(sql.Result), onNewUncommittedDml func())) { +func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onNewUncommittedDml func())) { logger := log.With("backend", "cache") - var totalAffectedByDeletes int64 var onNewUncommittedDmlCallsSinceLastTx int tx, err := cache.Beginx() @@ -254,13 +247,6 @@ func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onDeleted func(sql.Resul do( &tx, - func(result sql.Result) { // onDeleted - if affected, err := result.RowsAffected(); err == nil { - totalAffectedByDeletes += affected - } else { - log.Errorf("%+v", errors.Wrap(err, "can't get affected rows")) - } - }, func() { // onNewUncommittedDml onNewUncommittedDmlCallsSinceLastTx++ if onNewUncommittedDmlCallsSinceLastTx == bulk { @@ -303,19 +289,9 @@ func cacheSelect(cacheTx *sqlx.Tx, dest interface{}, query string, args ...inter } } -// cacheExec does cache.Exec(dml, args...). On non-recoverable errors the whole program exits if !allowFailure. -func cacheExec(cache sqlx.Execer, allowFailure bool, dml string, args ...interface{}) sql.Result { - res, err := cache.Exec(dml, args...) - if err != nil { - logger := log.With("backend", "cache", "dml", dml, "args", args) - - level := logger.Fatalf - if allowFailure { - level = logger.Errorf - } - - level("%+v", errors.Wrap(err, "can't perform DML")) +// cacheExec does cacheTx.Exec(dml, args...). On non-recoverable errors the whole program exits. +func cacheExec(cacheTx *sqlx.Tx, dml string, args ...interface{}) { + if _, err := cacheTx.Exec(dml, args...); err != nil { + log.With("backend", "cache", "dml", dml, "args", args).Fatalf("%+v", errors.Wrap(err, "can't perform DML")) } - - return res } From 0e9c93cd27a106849f3bf922af5f4422f1a18461 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 13 Jun 2022 15:55:28 +0200 Subject: [PATCH 061/103] cmd/ido2icingadb: rename onNewUncommittedDml to commitPeriodically --- cmd/ido2icingadb/cache.go | 49 ++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 39546211..b2a9e1b4 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -32,7 +32,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { ObjectId uint64 } - chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onNewUncommittedDml func()) { + chunkCacheTx(ht.cache, func(tx **sqlx.Tx, commitPeriodically func()) { var checkpoint struct { Cnt int64 MaxId sql.NullInt64 @@ -87,7 +87,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { ) } - onNewUncommittedDml() + commitPeriodically() checkpoint = idoRow.Id } @@ -118,7 +118,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { LastHardState uint8 } - chunkCacheTx(ht.cache, func(tx **sqlx.Tx, onNewUncommittedDml func()) { + chunkCacheTx(ht.cache, func(tx **sqlx.Tx, commitPeriodically func()) { var nextIds struct { Cnt int64 MinId sql.NullInt64 @@ -211,7 +211,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { ) } - onNewUncommittedDml() + commitPeriodically() checkpoint = idoRow.Id } @@ -233,38 +233,35 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { ht.bar.SetTotal(ht.bar.Current(), true) } -// chunkCacheTx rationale: during do operate on cache via *tx. On every completed operation call onNewUncommittedDml() +// chunkCacheTx rationale: during do operate on cache via *tx. After every completed operation call commitPeriodically() // which periodically commits *tx and starts a new tx. (That's why tx is a **, not just a *.) // (On non-recoverable errors the whole program exits.) -func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, onNewUncommittedDml func())) { +func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, commitPeriodically func())) { logger := log.With("backend", "cache") - var onNewUncommittedDmlCallsSinceLastTx int + var callsSinceLastTx int tx, err := cache.Beginx() if err != nil { logger.Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) } - do( - &tx, - func() { // onNewUncommittedDml - onNewUncommittedDmlCallsSinceLastTx++ - if onNewUncommittedDmlCallsSinceLastTx == bulk { - if err := tx.Commit(); err != nil { - logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) - } - - var err error - - tx, err = cache.Beginx() - if err != nil { - logger.Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) - } - - onNewUncommittedDmlCallsSinceLastTx = 0 + do(&tx, func() { // commitPeriodically + callsSinceLastTx++ + if callsSinceLastTx == bulk { + if err := tx.Commit(); err != nil { + logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } - }, - ) + + var err error + + tx, err = cache.Beginx() + if err != nil { + logger.Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) + } + + callsSinceLastTx = 0 + } + }) if err := tx.Commit(); err != nil { logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) From f40a39415cf9636901397f276bc4103344f666c4 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 20 Jun 2022 16:11:00 +0200 Subject: [PATCH 062/103] cmd/ido2icingadb: don't unnecessarily pre-fill ido_migration_progress --- cmd/ido2icingadb/main.go | 25 ++++++++++--------------- cmd/ido2icingadb/misc.go | 4 ++-- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index f647b0bd..4e20674f 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -199,19 +199,12 @@ func computeProgress(idb *icingadb.DB) { log.Fatalf("%+v", errors.Wrap(err, "can't create table ido_migration_progress")) } - stmt, _ := idb.BuildUpsertStmt(&IdoMigrationProgress{}) - types.forEach(func(ht *historyType) { - row := &IdoMigrationProgress{IdoMigrationProgressUpserter{ht.name}, 0} - - if _, err := idb.NamedExec(stmt, row); err != nil { - log.With("backend", "Icinga DB", "dml", stmt, "args", []interface{}{ht.name, 0}). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) - } - var query = idb.Rebind("SELECT last_ido_id FROM ido_migration_progress WHERE history_type=?") - if err := idb.Get(&ht.lastId, query, ht.name); err != nil { + switch err := idb.Get(&ht.lastId, query, ht.name); err { + case nil, sql.ErrNoRows: + default: log.With("backend", "Icinga DB", "query", query, "args", []interface{}{ht.name}). Fatalf("%+v", errors.Wrap(err, "can't perform query")) } @@ -336,6 +329,7 @@ func migrateOneType[IdoRow any]( icingaDbInserts := map[reflect.Type]string{} icingaDbUpdates := map[reflect.Type]string{} + upsertProgress, _ := idb.BuildUpsertStmt(&IdoMigrationProgress{}) ht.bar.SetCurrent(ht.done) @@ -398,12 +392,13 @@ func migrateOneType[IdoRow any]( } if lastIdoId != nil { - const stmt = "UPDATE ido_migration_progress SET last_ido_id=:last_ido_id " + - "WHERE history_type=:history_type" - args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} - if _, err := tx.NamedExec(stmt, args); err != nil { - log.With("backend", "Icinga DB", "dml", stmt, "args", args). + + _, err := tx.NamedExec(upsertProgress, &IdoMigrationProgress{ + IdoMigrationProgressUpserter{lastIdoId}, ht.name, + }) + if err != nil { + log.With("backend", "Icinga DB", "dml", upsertProgress, "args", args). Fatalf("%+v", errors.Wrap(err, "can't perform DML")) } } diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index a69db2ae..7867264e 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -67,7 +67,7 @@ func (e *eta) Decor(s decor.Statistics) string { var _ decor.Decorator = (*eta)(nil) type IdoMigrationProgressUpserter struct { - HistoryType string `json:"history_type"` + LastIdoId any `json:"last_ido_id"` } // Upsert implements the contracts.Upserter interface. @@ -77,7 +77,7 @@ func (impu *IdoMigrationProgressUpserter) Upsert() interface{} { type IdoMigrationProgress struct { IdoMigrationProgressUpserter `json:",inline"` - LastIdoId uint64 `json:"last_ido_id"` + HistoryType string `json:"history_type"` } // Assert interface compliance. From 752a9090abebc3f4746de7a4bc8d5053ee97395c Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 20 Jun 2022 19:09:12 +0200 Subject: [PATCH 063/103] cmd/ido2icingadb: improve code docs --- cmd/ido2icingadb/convert.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index f16ba31d..3ede3a21 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -592,7 +592,10 @@ func convertFlappingRows( return } -// notificationTypes maps IDO values to Icinga DB ones. +// notificationTypes maps IDO values[1] to Icinga DB ones[2]. +// +// [1]: https://github.com/Icinga/icinga2/blob/32c7f7730db154ba0dff5856a8985d125791c/lib/db_ido/dbevents.cpp#L1507-L1524 +// [2]: https://github.com/Icinga/icingadb/blob/8f31ac143875498797725adb9bfacf3d4/pkg/types/notification_type.go#L53-L61 var notificationTypes = map[uint8]icingadbTypes.NotificationType{5: 1, 6: 2, 7: 4, 8: 8, 1: 16, 2: 128, 3: 256} type notificationRow = struct { @@ -666,10 +669,11 @@ func convertNotificationRows( continue } - // The IDO tracks only sent notifications, but not notification config objects. We have to improvise. - name := strings.Join( - []string{row.Name1, row.Name2, "migrated from IDO", strconv.FormatUint(row.NotificationId, 36)}, "!", - ) + // The IDO tracks only sent notifications, but not notification config objects, nor even their names. + // We have to improvise. By the way we avoid unwanted collisions between synced and migrated data via "ID" + // instead of "HOST[!SERVICE]!NOTIFICATION" (ok as this name won't be parsed, but only hashed) and between + // migrated data itself via the history ID as object name, i.e. one "virtual object" per sent notification. + name := strconv.FormatUint(row.NotificationId, 10) var nt icingadbTypes.NotificationType if row.NotificationReason == 0 { From d9eb3689b8c8d393f8f834991cb2ea3ba910236f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 20 Jun 2022 19:10:09 +0200 Subject: [PATCH 064/103] cmd/ido2icingadb: fix downtime SLA end time --- cmd/ido2icingadb/convert.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index 3ede3a21..35ed01be 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -324,7 +324,7 @@ func convertDowntimeRows( triggerTime := convertTime(row.TriggerTime, 0) actualStart := convertTime(row.ActualStartTime, row.ActualStartTimeUsec) actualEnd := convertTime(row.ActualEndTime, row.ActualEndTimeUsec) - var startTime, endTime, cancelTime, slaEndTime icingadbTypes.UnixMilli + var startTime, endTime, cancelTime icingadbTypes.UnixMilli if scheduledEnd.Time().IsZero() { scheduledEnd = icingadbTypes.UnixMilli(scheduledStart.Time().Add(time.Duration(row.Duration) * time.Second)) @@ -346,10 +346,8 @@ func convertDowntimeRows( triggerTime = startTime } - slaEndTime = endTime if row.WasCancelled != 0 { cancelTime = actualEnd - slaEndTime = cancelTime } downtimeHistory = append(downtimeHistory, &history.DowntimeHistory{ @@ -429,7 +427,7 @@ func convertDowntimeRows( DowntimeStart: startTime, HasBeenCancelled: icingadbTypes.Bool{Bool: row.WasCancelled != 0, Valid: true}, CancelTime: cancelTime, - EndTime: slaEndTime, + EndTime: endTime, } s.DowntimeEnd.History = s From 4f871ecd3dd742d6a1b8e839e32a4092bf430cd4 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 22 Jun 2022 12:25:52 +0200 Subject: [PATCH 065/103] cmd/ido2icingadb: bulk, not prepare, upserts for the sake of performance. --- cmd/ido2icingadb/main.go | 81 +++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 4e20674f..d9f68cea 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "fmt" "github.com/goccy/go-yaml" + "github.com/icinga/icingadb/pkg/com" "github.com/icinga/icingadb/pkg/config" "github.com/icinga/icingadb/pkg/driver" "github.com/icinga/icingadb/pkg/icingadb" @@ -22,7 +23,6 @@ import ( "golang.org/x/sync/errgroup" "os" "path" - "reflect" "sync" "time" ) @@ -327,8 +327,6 @@ func migrateOneType[IdoRow any]( args = map[string]interface{}{"cache_limit": limit} } - icingaDbInserts := map[reflect.Type]string{} - icingaDbUpdates := map[reflect.Type]string{} upsertProgress, _ := idb.BuildUpsertStmt(&IdoMigrationProgress{}) ht.bar.SetCurrent(ht.done) @@ -356,39 +354,52 @@ func migrateOneType[IdoRow any]( log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) } - for _, operation := range [...]struct { - data [][]interface{} - buildStmt func(subject interface{}) (stmt string, _ int) - stmtCache map[reflect.Type]string - }{{inserts, idb.BuildUpsertStmt, icingaDbInserts}, {updates, idb.BuildUpdateStmt, icingaDbUpdates}} { - for _, table := range operation.data { - if len(table) < 1 { - continue - } - - tRow := reflect.TypeOf(table[0]) - - query, ok := operation.stmtCache[tRow] - if !ok { - query, _ = operation.buildStmt(table[0]) - operation.stmtCache[tRow] = query - } - - stmt, err := tx.PrepareNamed(query) - if err != nil { - log.With("backend", "Icinga DB", "dml", query). - Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) - } - - for _, row := range table { - if _, err := stmt.Exec(row); err != nil { - log.With("backend", "Icinga DB", "dml", query, "args", row). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) - } - } - - _ = stmt.Close() + for _, table := range inserts { + if len(table) < 1 { + continue } + + query, placeholders := idb.BuildUpsertStmt(table[0]) + + ch := make(chan any, len(table)) + for _, row := range table { + ch <- row + } + + close(ch) + + bulk := com.Bulk( + context.Background(), ch, idb.BatchSizeByPlaceholders(placeholders), com.NeverSplit[any], + ) + for rows := range bulk { + if _, err := tx.NamedExec(query, rows); err != nil { + log.With("backend", "Icinga DB", "dml", query, "args", rows). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + } + + for _, table := range updates { + if len(table) < 1 { + continue + } + + query, _ := idb.BuildUpdateStmt(table[0]) + + stmt, err := tx.PrepareNamed(query) + if err != nil { + log.With("backend", "Icinga DB", "dml", query). + Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) + } + + for _, row := range table { + if _, err := stmt.Exec(row); err != nil { + log.With("backend", "Icinga DB", "dml", query, "args", row). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } + } + + _ = stmt.Close() } if lastIdoId != nil { From fe24c2d3a0772ef2dc906a3829bb66f4cbd4c7bc Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 22 Jun 2022 12:27:02 +0200 Subject: [PATCH 066/103] cmd/ido2icingadb: fix missing config defaults --- cmd/ido2icingadb/main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index d9f68cea..818ed110 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -7,6 +7,7 @@ import ( _ "embed" "encoding/hex" "fmt" + "github.com/creasty/defaults" "github.com/goccy/go-yaml" "github.com/icinga/icingadb/pkg/com" "github.com/icinga/icingadb/pkg/config" @@ -104,6 +105,11 @@ func parseConfig(f *Flags) (_ *Config, exit int) { defer func() { _ = cf.Close() }() c := &Config{} + if err := defaults.Set(c); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "can't set config defaults: %s\n", err.Error()) + return nil, 2 + } + if err := yaml.NewDecoder(cf).Decode(c); err != nil { _, _ = fmt.Fprintf(os.Stderr, "can't parse config file: %s\n", err.Error()) return nil, 2 From f944844fb55b92eb6fdfe9e118b6171a78ee8522 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Jul 2022 12:14:43 +0200 Subject: [PATCH 067/103] cmd/ido2icingadb: build fix --- cmd/ido2icingadb/convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index 35ed01be..16ca4ef4 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -842,7 +842,7 @@ func convertStateRows( HardState: row.LastHardState, PreviousSoftState: row.LastState, PreviousHardState: previousHardState, - Attempt: uint8(row.CurrentCheckAttempt), + CheckAttempt: uint8(row.CurrentCheckAttempt), Output: icingadbTypes.String{NullString: row.Output}, LongOutput: icingadbTypes.String{NullString: row.LongOutput}, MaxCheckAttempts: uint32(row.MaxCheckAttempts), From 7896bc5024a6f9bbcea16ee27ebeca6e40e58dcc Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Jul 2022 15:41:21 +0200 Subject: [PATCH 068/103] cmd/ido2icingadb: fix missing flapping end events --- cmd/ido2icingadb/convert.go | 4 ++-- cmd/ido2icingadb/misc.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index 16ca4ef4..a5fed466 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -442,8 +442,8 @@ func convertDowntimeRows( // FlappingEnd updates an already migrated start event with the end event info. type FlappingEnd struct { - Id icingadbTypes.Binary - EndTime icingadbTypes.UnixMilli + Id icingadbTypes.Binary `json:"id"` + EndTime icingadbTypes.UnixMilli `json:"end_time"` } // Assert interface compliance. diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 7867264e..8ca3a9c8 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -295,7 +295,7 @@ var types = historyTypes{ cacheFiller: func(ht *historyType) { buildEventTimeCache(ht, []string{ "xh.flappinghistory_id id", "UNIX_TIMESTAMP(xh.event_time) event_time", - "xh.event_time_usec", "xh.event_type-1000 event_is_start", "xh.object_id", + "xh.event_time_usec", "1001-xh.event_type event_is_start", "xh.object_id", }) }, migrationQuery: flappingMigrationQuery, From 90bc748aec0d032ba85ee1eef247ac97c86f8d5b Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Jul 2022 16:50:12 +0200 Subject: [PATCH 069/103] cmd/ido2icingadb: indicate no notification author as "", not "-" It looks nicer in Icinga DB Web. --- cmd/ido2icingadb/convert.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index a5fed466..893ef2f5 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -721,7 +721,6 @@ func convertNotificationRows( SendTime: ts, State: row.State, PreviousHardState: previousHardState, - Author: "-", Text: icingadbTypes.String{ NullString: sql.NullString{ String: text, From 9601b1685f29b72c1ec083e6d7f74f6f0f65e7d7 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 13 Jul 2022 11:59:32 +0200 Subject: [PATCH 070/103] cmd/ido2icingadb: fix missing ack clearings --- cmd/ido2icingadb/convert.go | 351 +++++++++++---------------- cmd/ido2icingadb/embed/ack_query.sql | 8 - cmd/ido2icingadb/main.go | 5 +- cmd/ido2icingadb/misc.go | 20 +- 4 files changed, 147 insertions(+), 237 deletions(-) delete mode 100644 cmd/ido2icingadb/embed/ack_query.sql diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index 893ef2f5..6868015d 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -15,9 +15,6 @@ import ( "time" ) -//go:embed embed/ack_query.sql -var acknowledgementMigrationQuery string - //go:embed embed/comment_query.sql var commentMigrationQuery string @@ -33,160 +30,6 @@ var notificationMigrationQuery string //go:embed embed/state_query.sql var stateMigrationQuery string -// AckClear updates an already migrated ack event with the clear event info. -type AckClear struct { - Id icingadbTypes.Binary - ClearTime icingadbTypes.UnixMilli -} - -// Assert interface compliance. -var _ contracts.TableNamer = (*AckClear)(nil) - -// TableName implements the contracts.TableNamer interface. -func (*AckClear) TableName() string { - return "acknowledgement_history" -} - -type acknowledgementRow = struct { - AcknowledgementId uint64 - EntryTime int64 - EntryTimeUsec uint32 - AcknowledgementType uint8 - AuthorName sql.NullString - CommentData sql.NullString - IsSticky uint8 - PersistentComment uint8 - EndTime sql.NullInt64 - ObjecttypeId uint8 - Name1 string - Name2 string -} - -func convertAcknowledgementRows( - env string, envId, endpointId icingadbTypes.Binary, - selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []acknowledgementRow, -) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) { - if len(idoRows) < 1 { - return - } - - var cached []struct { - HistoryId uint64 - EventTime int64 - EventTimeUsec uint32 - } - selectCache( - &cached, "SELECT history_id, event_time, event_time_usec FROM end_start_time WHERE history_id BETWEEN ? AND ?", - idoRows[0].AcknowledgementId, idoRows[len(idoRows)-1].AcknowledgementId, - ) - - // Needed for set time (see below). - cachedById := make(map[uint64]icingadbTypes.UnixMilli, len(cached)) - for _, c := range cached { - cachedById[c.HistoryId] = convertTime(c.EventTime, c.EventTimeUsec) - } - - var acknowledgementHistory, acknowledgementHistoryUpdates, allHistory []interface{} - for _, row := range idoRows { - ts := convertTime(row.EntryTime, row.EntryTimeUsec) - - // Needed for ID (see below). - var set icingadbTypes.UnixMilli - if row.AcknowledgementType == 0 { // clear - var ok bool - set, ok = cachedById[row.AcknowledgementId] - - if !ok { - continue - } - } else { - set = ts - } - - name := row.Name1 - if row.Name2 != "" { - name += "!" + row.Name2 - } - - typ := objectTypes[row.ObjecttypeId] - hostId := calcObjectId(env, row.Name1) - serviceId := calcServiceId(env, row.Name1, row.Name2) - setTime := float64(set.Time().UnixMilli()) - acknowledgementHistoryId := hashAny([]interface{}{env, name, setTime}) - - if row.AcknowledgementType == 0 { // clear - // The set counterpart should already have been inserted. - acknowledgementHistoryUpdates = append(acknowledgementHistoryUpdates, &AckClear{ - acknowledgementHistoryId, ts, - }) - - h := &history.HistoryAck{ - HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{ - Id: hashAny([]interface{}{env, "ack_clear", name, setTime}), - }, - EnvironmentId: envId, - EndpointId: endpointId, - ObjectType: typ, - HostId: hostId, - ServiceId: serviceId, - EventType: "ack_clear", - }, - AcknowledgementHistoryId: acknowledgementHistoryId, - SetTime: set, - ClearTime: ts, - } - - h.EventTime.History = h - allHistory = append(allHistory, h) - } else { // set - acknowledgementHistory = append(acknowledgementHistory, &history.AcknowledgementHistory{ - EntityWithoutChecksum: v1.EntityWithoutChecksum{ - IdMeta: v1.IdMeta{Id: acknowledgementHistoryId}, - }, - HistoryTableMeta: history.HistoryTableMeta{ - EnvironmentId: envId, - EndpointId: endpointId, - ObjectType: typ, - HostId: hostId, - ServiceId: serviceId, - }, - SetTime: set, - Author: icingadbTypes.String{NullString: row.AuthorName}, - Comment: icingadbTypes.String{NullString: row.CommentData}, - ExpireTime: convertTime(row.EndTime.Int64, 0), - IsPersistent: icingadbTypes.Bool{Bool: row.PersistentComment != 0, Valid: true}, - IsSticky: icingadbTypes.Bool{Bool: row.IsSticky != 0, Valid: true}, - }) - - h := &history.HistoryAck{ - HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{ - Id: hashAny([]interface{}{env, "ack_set", name, setTime}), - }, - EnvironmentId: envId, - EndpointId: endpointId, - ObjectType: typ, - HostId: hostId, - ServiceId: serviceId, - EventType: "ack_set", - }, - AcknowledgementHistoryId: acknowledgementHistoryId, - SetTime: set, - } - - h.EventTime.History = h - allHistory = append(allHistory, h) - } - - checkpoint = row.AcknowledgementId - } - - icingaDbUpdates = [][]interface{}{acknowledgementHistoryUpdates} - icingaDbInserts = [][]interface{}{acknowledgementHistory, allHistory} - return -} - type commentRow = struct { CommenthistoryId uint64 EntryTime int64 @@ -208,81 +51,169 @@ func convertCommentRows( env string, envId, endpointId icingadbTypes.Binary, _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []commentRow, ) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { - var commentHistory, allHistory []interface{} + var commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck []any for _, row := range idoRows { - id := calcObjectId(env, row.Name) typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) serviceId := calcServiceId(env, row.Name1, row.Name2) - entryTime := convertTime(row.EntryTime, row.EntryTimeUsec) - removeTime := convertTime(row.DeletionTime, row.DeletionTimeUsec) - expireTime := convertTime(row.ExpirationTime, 0) - commentHistory = append(commentHistory, &history.CommentHistory{ - CommentHistoryEntity: history.CommentHistoryEntity{CommentId: id}, - HistoryTableMeta: history.HistoryTableMeta{ - EnvironmentId: envId, - EndpointId: endpointId, - ObjectType: typ, - HostId: hostId, - ServiceId: serviceId, - }, - CommentHistoryUpserter: history.CommentHistoryUpserter{ - RemoveTime: removeTime, - HasBeenRemoved: icingadbTypes.Bool{Bool: !removeTime.Time().IsZero(), Valid: true}, - }, - EntryTime: entryTime, - Author: row.AuthorName, - Comment: row.CommentData, - EntryType: icingadbTypes.CommentType(row.EntryType), - IsPersistent: icingadbTypes.Bool{Bool: row.IsPersistent != 0, Valid: true}, - IsSticky: icingadbTypes.Bool{Bool: false, Valid: true}, - ExpireTime: expireTime, - }) + switch row.EntryType { + case 1: // user + id := calcObjectId(env, row.Name) + entryTime := convertTime(row.EntryTime, row.EntryTimeUsec) + removeTime := convertTime(row.DeletionTime, row.DeletionTimeUsec) + expireTime := convertTime(row.ExpirationTime, 0) - h1 := &history.HistoryComment{ - HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: hashAny([]string{env, "comment_add", row.Name})}, - EnvironmentId: envId, - EndpointId: endpointId, - ObjectType: typ, - HostId: hostId, - ServiceId: serviceId, - EventType: "comment_add", - }, - CommentHistoryId: id, - EntryTime: entryTime, - } - - h1.EventTime.History = h1 - allHistory = append(allHistory, h1) - - if !removeTime.Time().IsZero() { // remove - h2 := &history.HistoryComment{ - HistoryMeta: history.HistoryMeta{ - HistoryEntity: history.HistoryEntity{Id: hashAny([]string{env, "comment_remove", row.Name})}, + commentHistory = append(commentHistory, &history.CommentHistory{ + CommentHistoryEntity: history.CommentHistoryEntity{CommentId: id}, + HistoryTableMeta: history.HistoryTableMeta{ EnvironmentId: envId, EndpointId: endpointId, ObjectType: typ, HostId: hostId, ServiceId: serviceId, - EventType: "comment_remove", + }, + CommentHistoryUpserter: history.CommentHistoryUpserter{ + RemoveTime: removeTime, + HasBeenRemoved: icingadbTypes.Bool{Bool: !removeTime.Time().IsZero(), Valid: true}, + }, + EntryTime: entryTime, + Author: row.AuthorName, + Comment: row.CommentData, + EntryType: icingadbTypes.CommentType(row.EntryType), + IsPersistent: icingadbTypes.Bool{Bool: row.IsPersistent != 0, Valid: true}, + IsSticky: icingadbTypes.Bool{Bool: false, Valid: true}, + ExpireTime: expireTime, + }) + + h1 := &history.HistoryComment{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: hashAny([]string{env, "comment_add", row.Name})}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "comment_add", }, CommentHistoryId: id, EntryTime: entryTime, - RemoveTime: removeTime, - ExpireTime: expireTime, } - h2.EventTime.History = h2 - allHistory = append(allHistory, h2) + h1.EventTime.History = h1 + allHistoryComment = append(allHistoryComment, h1) + + if !removeTime.Time().IsZero() { // remove + h2 := &history.HistoryComment{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{Id: hashAny([]string{env, "comment_remove", row.Name})}, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "comment_remove", + }, + CommentHistoryId: id, + EntryTime: entryTime, + RemoveTime: removeTime, + ExpireTime: expireTime, + } + + h2.EventTime.History = h2 + allHistoryComment = append(allHistoryComment, h2) + } + case 4: // ack + name := row.Name1 + if row.Name2 != "" { + name += "!" + row.Name2 + } + + setTime := convertTime(row.EntryTime, row.EntryTimeUsec) + setTs := float64(setTime.Time().UnixMilli()) + clearTime := convertTime(row.DeletionTime, row.DeletionTimeUsec) + acknowledgementHistoryId := hashAny([]any{env, name, setTs}) + + acknowledgementHistory = append(acknowledgementHistory, &history.AcknowledgementHistory{ + EntityWithoutChecksum: v1.EntityWithoutChecksum{ + IdMeta: v1.IdMeta{Id: acknowledgementHistoryId}, + }, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + AckHistoryUpserter: history.AckHistoryUpserter{ClearTime: clearTime}, + SetTime: setTime, + Author: icingadbTypes.String{ + NullString: sql.NullString{ + String: row.AuthorName, + Valid: true, + }, + }, + Comment: icingadbTypes.String{ + NullString: sql.NullString{ + String: row.CommentData, + Valid: true, + }, + }, + ExpireTime: convertTime(row.ExpirationTime, 0), + IsPersistent: icingadbTypes.Bool{ + Bool: row.IsPersistent != 0, + Valid: true, + }, + }) + + h1 := &history.HistoryAck{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{ + Id: hashAny([]any{env, "ack_set", name, setTs}), + }, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "ack_set", + }, + AcknowledgementHistoryId: acknowledgementHistoryId, + SetTime: setTime, + ClearTime: clearTime, + } + + h1.EventTime.History = h1 + allHistoryAck = append(allHistoryAck, h1) + + if !clearTime.Time().IsZero() { + h2 := &history.HistoryAck{ + HistoryMeta: history.HistoryMeta{ + HistoryEntity: history.HistoryEntity{ + Id: hashAny([]any{env, "ack_clear", name, setTs}), + }, + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + EventType: "ack_clear", + }, + AcknowledgementHistoryId: acknowledgementHistoryId, + SetTime: setTime, + ClearTime: clearTime, + } + + h2.EventTime.History = h2 + allHistoryAck = append(allHistoryAck, h2) + } } checkpoint = row.CommenthistoryId } - icingaDbInserts = [][]interface{}{commentHistory, allHistory} + icingaDbInserts = [][]any{commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck} return } diff --git a/cmd/ido2icingadb/embed/ack_query.sql b/cmd/ido2icingadb/embed/ack_query.sql deleted file mode 100644 index 4ba2a441..00000000 --- a/cmd/ido2icingadb/embed/ack_query.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT ah.acknowledgement_id, UNIX_TIMESTAMP(ah.entry_time) entry_time, ah.entry_time_usec, - ah.acknowledgement_type, ah.author_name, ah.comment_data, ah.is_sticky, ah.persistent_comment, - UNIX_TIMESTAMP(ah.end_time) end_time, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 -FROM icinga_acknowledgements ah USE INDEX (PRIMARY) -INNER JOIN icinga_objects o ON o.object_id=ah.object_id -WHERE ah.acknowledgement_id > :checkpoint -- where we were interrupted -ORDER BY ah.acknowledgement_id -- this way we know what has already been migrated from just the last row's ID -LIMIT :bulk diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 818ed110..243106ab 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -24,6 +24,7 @@ import ( "golang.org/x/sync/errgroup" "os" "path" + "regexp" "sync" "time" ) @@ -118,6 +119,8 @@ func parseConfig(f *Flags) (_ *Config, exit int) { return c, -1 } +var nonWords = regexp.MustCompile(`\W+`) + // mkCache ensures /.sqlite3 files are present and contain their schema // and initializes types[*].cache. (On non-recoverable errors the whole program exits.) func mkCache(f *Flags, mapper *reflectx.Mapper) { @@ -132,7 +135,7 @@ func mkCache(f *Flags, mapper *reflectx.Mapper) { return } - file := path.Join(f.Cache, ht.name+".sqlite3") + file := path.Join(f.Cache, nonWords.ReplaceAllLiteralString(ht.name, "_")+".sqlite3") var err error ht.cache, err = sqlx.Open("sqlite3", "file:"+file) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 8ca3a9c8..f358ef1d 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -187,7 +187,7 @@ func sliceIdoHistory[Row any]( // historyType specifies a history data type. type historyType struct { - // name is a human-readable, but machine-friendly common name. + // name is a human-readable common name. name string // idoTable specifies the source table. idoTable string @@ -254,23 +254,7 @@ func (hts historyTypes) forEach(f func(*historyType)) { var types = historyTypes{ { - name: "acknowledgement", - idoTable: "icinga_acknowledgements", - idoIdColumn: "acknowledgement_id", - cacheSchema: eventTimeCacheSchema, - cacheFiller: func(ht *historyType) { - buildEventTimeCache(ht, []string{ - "xh.acknowledgement_id id", "UNIX_TIMESTAMP(xh.entry_time) event_time", - "xh.entry_time_usec event_time_usec", "xh.acknowledgement_type event_is_start", "xh.object_id", - }) - }, - migrationQuery: acknowledgementMigrationQuery, - migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { - migrateOneType(c, idb, envId, endpId, idbTx, ht, convertAcknowledgementRows) - }, - }, - { - name: "comment", + name: "ack & comment", idoTable: "icinga_commenthistory", idoIdColumn: "commenthistory_id", migrationQuery: commentMigrationQuery, From d69ccd13d7d8edc6b60bfacb735b924d471b011f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 13 Jul 2022 12:52:26 +0200 Subject: [PATCH 071/103] cmd/ido2icingadb: don't unnecessarily overwrite already written/migrated history --- cmd/ido2icingadb/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 243106ab..b610c8fe 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -368,7 +368,7 @@ func migrateOneType[IdoRow any]( continue } - query, placeholders := idb.BuildUpsertStmt(table[0]) + query, placeholders := idb.BuildInsertIgnoreStmt(table[0]) ch := make(chan any, len(table)) for _, row := range table { From 74fce9b3dee610cd7922accf9ceae7df79ef6abf Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 13 Jul 2022 17:57:32 +0200 Subject: [PATCH 072/103] cmd/ido2icingadb: fix missing flapping_history#percent_state_change_end --- cmd/ido2icingadb/convert.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index 6868015d..ae31be41 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -373,8 +373,9 @@ func convertDowntimeRows( // FlappingEnd updates an already migrated start event with the end event info. type FlappingEnd struct { - Id icingadbTypes.Binary `json:"id"` - EndTime icingadbTypes.UnixMilli `json:"end_time"` + Id icingadbTypes.Binary `json:"id"` + EndTime icingadbTypes.UnixMilli `json:"end_time"` + PercentStateChangeEnd icingadbTypes.Float `json:"percent_state_change_end"` } // Assert interface compliance. @@ -452,7 +453,11 @@ func convertFlappingRows( if row.EventType == 1001 { // end // The start counterpart should already have been inserted. - flappingHistoryUpdates = append(flappingHistoryUpdates, &FlappingEnd{flappingHistoryId, ts}) + flappingHistoryUpdates = append(flappingHistoryUpdates, &FlappingEnd{ + Id: flappingHistoryId, + EndTime: ts, + PercentStateChangeEnd: icingadbTypes.Float{NullFloat64: row.PercentStateChange}, + }) h := &history.HistoryFlapping{ HistoryMeta: history.HistoryMeta{ From 1caca0b8b8519ddd2ce396ee044ccad9856c23af Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 13 Jul 2022 17:58:38 +0200 Subject: [PATCH 073/103] cmd/ido2icingadb: fix duplicate comment --- cmd/ido2icingadb/convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index ae31be41..eeccdc99 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -478,7 +478,7 @@ func convertFlappingRows( h.EventTime.History = h allHistory = append(allHistory, h) - } else { // end + } else { flappingHistory = append(flappingHistory, &history.FlappingHistory{ EntityWithoutChecksum: v1.EntityWithoutChecksum{ IdMeta: v1.IdMeta{Id: flappingHistoryId}, From 98bb53b2f7a3eb2a956d1f529dc277335d4eca4f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 19 Jul 2022 15:20:07 +0200 Subject: [PATCH 074/103] cmd/ido2icingadb: clean up cache --- cmd/ido2icingadb/main.go | 19 +++++++++++++++++++ cmd/ido2icingadb/misc.go | 4 +++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index b610c8fe..ad4eef7e 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -94,6 +94,9 @@ func main() { log.Info("Actually migrating") migrate(c, idb, envId) + + log.Info("Cleaning up cache") + cleanupCache() } // parseConfig validates the f.Config file and returns the config and -1 or - on failure - nil and an exit code. @@ -143,6 +146,7 @@ func mkCache(f *Flags, mapper *reflectx.Mapper) { log.With("file", file).Fatalf("%+v", errors.Wrap(err, "can't open SQLite database")) } + ht.cacheFile = file ht.cache.Mapper = mapper if _, err := ht.cache.Exec(ht.cacheSchema); err != nil { @@ -434,3 +438,18 @@ func migrateOneType[IdoRow any]( ht.bar.SetTotal(ht.bar.Current(), true) } + +// cleanupCache removes /.sqlite3 files. +func cleanupCache() { + types.forEach(func(ht *historyType) { + if ht.cacheFile != "" { + if err := ht.cache.Close(); err != nil { + log.With("file", ht.cacheFile).Warnf("%+v", errors.Wrap(err, "can't close SQLite database")) + } + + if err := os.Remove(ht.cacheFile); err != nil { + log.With("file", ht.cacheFile).Warnf("%+v", errors.Wrap(err, "can't remove SQLite database")) + } + } + }) +} diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index f358ef1d..1f6ef455 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -205,7 +205,9 @@ type historyType struct { migrate func(c *Config, idb *icingadb.DB, envId []byte, endpointId [sha1.Size]byte, idbTx *sync.Mutex, ht *historyType) - // cache represents .sqlite3. + // cacheFile locates .sqlite3. + cacheFile string + // cache represents . cache *sqlx.DB // snapshot represents the data source. snapshot *sqlx.Tx From f063687b2b98f3faaff96c25bd6c163ccd4040bc Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Jul 2022 10:58:50 +0200 Subject: [PATCH 075/103] DB#BuildInsertIgnoreStmt(): handle primary key being not "id" --- pkg/icingadb/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/icingadb/db.go b/pkg/icingadb/db.go index 33cbb990..bbd02e6d 100644 --- a/pkg/icingadb/db.go +++ b/pkg/icingadb/db.go @@ -131,7 +131,7 @@ func (db *DB) BuildInsertIgnoreStmt(into interface{}) (string, int) { switch db.DriverName() { case driver.MySQL: // MySQL treats UPDATE id = id as a no-op. - clause = "ON DUPLICATE KEY UPDATE id = id" + clause = fmt.Sprintf(`ON DUPLICATE KEY UPDATE "%s" = "%s"`, columns[0], columns[0]) case driver.PostgreSQL: clause = fmt.Sprintf("ON CONFLICT ON CONSTRAINT pk_%s DO NOTHING", table) } From cc98c34d18e26392a438b3809a9c384ada5b39bf Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Jul 2022 13:32:12 +0200 Subject: [PATCH 076/103] cmd/ido2icingadb: centralise notification type conversion --- cmd/ido2icingadb/convert.go | 51 ++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index eeccdc99..0125ec41 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -526,12 +526,6 @@ func convertFlappingRows( return } -// notificationTypes maps IDO values[1] to Icinga DB ones[2]. -// -// [1]: https://github.com/Icinga/icinga2/blob/32c7f7730db154ba0dff5856a8985d125791c/lib/db_ido/dbevents.cpp#L1507-L1524 -// [2]: https://github.com/Icinga/icingadb/blob/8f31ac143875498797725adb9bfacf3d4/pkg/types/notification_type.go#L53-L61 -var notificationTypes = map[uint8]icingadbTypes.NotificationType{5: 1, 6: 2, 7: 4, 8: 8, 1: 16, 2: 128, 3: 256} - type notificationRow = struct { NotificationId uint64 NotificationReason uint8 @@ -609,21 +603,11 @@ func convertNotificationRows( // migrated data itself via the history ID as object name, i.e. one "virtual object" per sent notification. name := strconv.FormatUint(row.NotificationId, 10) - var nt icingadbTypes.NotificationType - if row.NotificationReason == 0 { - if row.State == 0 { - nt = 64 // recovery - } else { - nt = 32 // problem - } - } else { - nt = notificationTypes[row.NotificationReason] - } + nt := convertNotificationType(row.NotificationReason, row.State) ntEnum, err := nt.Value() if err != nil { - // Programming error - panic(err) + continue } ts := convertTime(row.EndTime, row.EndTimeUsec) @@ -702,6 +686,37 @@ func convertNotificationRows( return } +// convertNotificationType maps IDO values[1] to Icinga DB ones[2]. +// +// [1]: https://github.com/Icinga/icinga2/blob/32c7f7730db154ba0dff5856a8985d125791c/lib/db_ido/dbevents.cpp#L1507-L1524 +// [2]: https://github.com/Icinga/icingadb/blob/8f31ac143875498797725adb9bfacf3d4/pkg/types/notification_type.go#L53-L61 +func convertNotificationType(notificationReason, state uint8) icingadbTypes.NotificationType { + switch notificationReason { + case 0: // state + if state == 0 { + return 64 // recovery + } else { + return 32 // problem + } + case 1: // acknowledgement + return 16 + case 2: // flapping start + return 128 + case 3: // flapping end + return 256 + case 5: // downtime start + return 1 + case 6: // downtime end + return 2 + case 7: // downtime removed + return 4 + case 8: // custom + return 8 + default: // bad notification type + return 0 + } +} + type stateRow = struct { StatehistoryId uint64 StateTime int64 From 58cfbf46b543210bf35ed308e9d0ff67934b309e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Jul 2022 15:31:38 +0200 Subject: [PATCH 077/103] cmd/ido2icingadb: allow converters to upsert --- cmd/ido2icingadb/convert.go | 10 ++++----- cmd/ido2icingadb/main.go | 43 +++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index 0125ec41..38e61f2e 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -50,7 +50,7 @@ type commentRow = struct { func convertCommentRows( env string, envId, endpointId icingadbTypes.Binary, _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []commentRow, -) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { +) (_, icingaDbInserts, _ [][]any, checkpoint any) { var commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck []any for _, row := range idoRows { @@ -242,7 +242,7 @@ type downtimeRow = struct { func convertDowntimeRows( env string, envId, endpointId icingadbTypes.Binary, _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []downtimeRow, -) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { +) (_, icingaDbInserts, _ [][]any, checkpoint any) { var downtimeHistory, allHistory, sla []interface{} for _, row := range idoRows { @@ -402,7 +402,7 @@ type flappingRow = struct { func convertFlappingRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []flappingRow, -) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}) { +) (icingaDbUpdates, icingaDbInserts, _ [][]any, checkpoint any) { if len(idoRows) < 1 { return } @@ -543,7 +543,7 @@ type notificationRow = struct { func convertNotificationRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []notificationRow, -) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { +) (_, icingaDbInserts, _ [][]any, checkpoint any) { if len(idoRows) < 1 { return } @@ -738,7 +738,7 @@ type stateRow = struct { func convertStateRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []stateRow, -) (_, icingaDbInserts [][]interface{}, checkpoint interface{}) { +) (_, icingaDbInserts, _ [][]any, checkpoint any) { if len(idoRows) < 1 { return } diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index ad4eef7e..adcbf2b3 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -295,7 +295,7 @@ func migrateOneType[IdoRow any]( c *Config, idb *icingadb.DB, envId []byte, endpointId [sha1.Size]byte, idbTx *sync.Mutex, ht *historyType, convertRows func(env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, - idoRows []IdoRow) (icingaDbUpdates, icingaDbInserts [][]interface{}, checkpoint interface{}), + idoRows []IdoRow) (icingaDbUpdates, icingaDbInserts, icingaDbUpserts [][]any, checkpoint any), ) { var lastQuery string var lastStmt *sqlx.Stmt @@ -349,7 +349,7 @@ func migrateOneType[IdoRow any]( ht.snapshot, ht.migrationQuery, args, ht.lastId, func(idoRows []IdoRow) (checkpoint interface{}) { // ... convert them, ... - updates, inserts, lastIdoId := convertRows( + updates, inserts, upserts, lastIdoId := convertRows( c.Icinga2.Env, envId, endpointId[:], selectCache, ht.snapshot, idoRows, ) @@ -367,27 +367,32 @@ func migrateOneType[IdoRow any]( log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) } - for _, table := range inserts { - if len(table) < 1 { - continue - } + for _, op := range []struct { + data [][]any + stmtBuilder func(any) (string, int) + }{{inserts, idb.BuildInsertIgnoreStmt}, {upserts, idb.BuildUpsertStmt}} { + for _, table := range op.data { + if len(table) < 1 { + continue + } - query, placeholders := idb.BuildInsertIgnoreStmt(table[0]) + query, placeholders := op.stmtBuilder(table[0]) - ch := make(chan any, len(table)) - for _, row := range table { - ch <- row - } + ch := make(chan any, len(table)) + for _, row := range table { + ch <- row + } - close(ch) + close(ch) - bulk := com.Bulk( - context.Background(), ch, idb.BatchSizeByPlaceholders(placeholders), com.NeverSplit[any], - ) - for rows := range bulk { - if _, err := tx.NamedExec(query, rows); err != nil { - log.With("backend", "Icinga DB", "dml", query, "args", rows). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + bulk := com.Bulk( + context.Background(), ch, idb.BatchSizeByPlaceholders(placeholders), com.NeverSplit[any], + ) + for rows := range bulk { + if _, err := tx.NamedExec(query, rows); err != nil { + log.With("backend", "Icinga DB", "dml", query, "args", rows). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) + } } } } From 34ef6bec44b9b70cba369e86c488ab9fdf79d05f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Jul 2022 15:38:35 +0200 Subject: [PATCH 078/103] cmd/ido2icingadb: upsert, not update, to make bulk statements --- cmd/ido2icingadb/convert.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index 38e61f2e..e01c9d3c 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -402,7 +402,7 @@ type flappingRow = struct { func convertFlappingRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []flappingRow, -) (icingaDbUpdates, icingaDbInserts, _ [][]any, checkpoint any) { +) (_, icingaDbInserts, icingaDbUpserts [][]any, checkpoint any) { if len(idoRows) < 1 { return } @@ -423,7 +423,7 @@ func convertFlappingRows( cachedById[c.HistoryId] = convertTime(c.EventTime, c.EventTimeUsec) } - var flappingHistory, flappingHistoryUpdates, allHistory []interface{} + var flappingHistory, flappingHistoryUpserts, allHistory []any for _, row := range idoRows { ts := convertTime(row.EventTime, row.EventTimeUsec) @@ -453,10 +453,24 @@ func convertFlappingRows( if row.EventType == 1001 { // end // The start counterpart should already have been inserted. - flappingHistoryUpdates = append(flappingHistoryUpdates, &FlappingEnd{ - Id: flappingHistoryId, - EndTime: ts, - PercentStateChangeEnd: icingadbTypes.Float{NullFloat64: row.PercentStateChange}, + flappingHistoryUpserts = append(flappingHistoryUpserts, &history.FlappingHistory{ + EntityWithoutChecksum: v1.EntityWithoutChecksum{ + IdMeta: v1.IdMeta{Id: flappingHistoryId}, + }, + HistoryTableMeta: history.HistoryTableMeta{ + EnvironmentId: envId, + EndpointId: endpointId, + ObjectType: typ, + HostId: hostId, + ServiceId: serviceId, + }, + FlappingHistoryUpserter: history.FlappingHistoryUpserter{ + EndTime: ts, + PercentStateChangeEnd: icingadbTypes.Float{NullFloat64: row.PercentStateChange}, + FlappingThresholdLow: float32(row.LowThreshold), + FlappingThresholdHigh: float32(row.HighThreshold), + }, + StartTime: start, }) h := &history.HistoryFlapping{ @@ -521,8 +535,8 @@ func convertFlappingRows( checkpoint = row.FlappinghistoryId } - icingaDbUpdates = [][]interface{}{flappingHistoryUpdates} icingaDbInserts = [][]interface{}{flappingHistory, allHistory} + icingaDbUpserts = [][]interface{}{flappingHistoryUpserts} return } From fcae6759d31f7a61d61f1b26655400c341c7a184 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Jul 2022 15:42:15 +0200 Subject: [PATCH 079/103] cmd/ido2icingadb: remove unused code --- cmd/ido2icingadb/convert.go | 10 +++++----- cmd/ido2icingadb/main.go | 27 ++------------------------- 2 files changed, 7 insertions(+), 30 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index e01c9d3c..df318e08 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -50,7 +50,7 @@ type commentRow = struct { func convertCommentRows( env string, envId, endpointId icingadbTypes.Binary, _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []commentRow, -) (_, icingaDbInserts, _ [][]any, checkpoint any) { +) (icingaDbInserts, _ [][]any, checkpoint any) { var commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck []any for _, row := range idoRows { @@ -242,7 +242,7 @@ type downtimeRow = struct { func convertDowntimeRows( env string, envId, endpointId icingadbTypes.Binary, _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []downtimeRow, -) (_, icingaDbInserts, _ [][]any, checkpoint any) { +) (icingaDbInserts, _ [][]any, checkpoint any) { var downtimeHistory, allHistory, sla []interface{} for _, row := range idoRows { @@ -402,7 +402,7 @@ type flappingRow = struct { func convertFlappingRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []flappingRow, -) (_, icingaDbInserts, icingaDbUpserts [][]any, checkpoint any) { +) (icingaDbInserts, icingaDbUpserts [][]any, checkpoint any) { if len(idoRows) < 1 { return } @@ -557,7 +557,7 @@ type notificationRow = struct { func convertNotificationRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []notificationRow, -) (_, icingaDbInserts, _ [][]any, checkpoint any) { +) (icingaDbInserts, _ [][]any, checkpoint any) { if len(idoRows) < 1 { return } @@ -752,7 +752,7 @@ type stateRow = struct { func convertStateRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []stateRow, -) (_, icingaDbInserts, _ [][]any, checkpoint any) { +) (icingaDbInserts, _ [][]any, checkpoint any) { if len(idoRows) < 1 { return } diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index adcbf2b3..f8de3848 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -295,7 +295,7 @@ func migrateOneType[IdoRow any]( c *Config, idb *icingadb.DB, envId []byte, endpointId [sha1.Size]byte, idbTx *sync.Mutex, ht *historyType, convertRows func(env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, - idoRows []IdoRow) (icingaDbUpdates, icingaDbInserts, icingaDbUpserts [][]any, checkpoint any), + idoRows []IdoRow) (icingaDbInserts, icingaDbUpserts [][]any, checkpoint any), ) { var lastQuery string var lastStmt *sqlx.Stmt @@ -349,7 +349,7 @@ func migrateOneType[IdoRow any]( ht.snapshot, ht.migrationQuery, args, ht.lastId, func(idoRows []IdoRow) (checkpoint interface{}) { // ... convert them, ... - updates, inserts, upserts, lastIdoId := convertRows( + inserts, upserts, lastIdoId := convertRows( c.Icinga2.Env, envId, endpointId[:], selectCache, ht.snapshot, idoRows, ) @@ -397,29 +397,6 @@ func migrateOneType[IdoRow any]( } } - for _, table := range updates { - if len(table) < 1 { - continue - } - - query, _ := idb.BuildUpdateStmt(table[0]) - - stmt, err := tx.PrepareNamed(query) - if err != nil { - log.With("backend", "Icinga DB", "dml", query). - Fatalf("%+v", errors.Wrap(err, "can't prepare DML")) - } - - for _, row := range table { - if _, err := stmt.Exec(row); err != nil { - log.With("backend", "Icinga DB", "dml", query, "args", row). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) - } - } - - _ = stmt.Close() - } - if lastIdoId != nil { args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} From 59c77eb90a18967a3219ed5ef263e636130bd522 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 26 Jul 2022 11:09:18 +0200 Subject: [PATCH 080/103] cmd/ido2icingadb: reduce bulk size --- cmd/ido2icingadb/misc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 1f6ef455..6717c106 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -86,7 +86,7 @@ var ( _ contracts.Upserter = (*IdoMigrationProgress)(nil) ) -const bulk = 10000 +const bulk = 100 // log is the root logger. var log = func() *zap.SugaredLogger { From f0ec8b01585fe41708fe2a351c4361b9e32da0c0 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 28 Jul 2022 16:46:07 +0200 Subject: [PATCH 081/103] cmd/ido2icingadb: allow to migrate multiple IDO databases into one Icinga DB database as separate envs. --- .../embed/ido_migration_progress_schema.sql | 7 ++++--- cmd/ido2icingadb/main.go | 14 +++++++++----- cmd/ido2icingadb/misc.go | 1 + 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql b/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql index aa682484..4049e6ae 100644 --- a/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql +++ b/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql @@ -1,6 +1,7 @@ CREATE TABLE IF NOT EXISTS ido_migration_progress ( - history_type VARCHAR(63) NOT NULL, - last_ido_id BIGINT NOT NULL, + environment_id CHAR(40) NOT NULL, -- Hex SHA1. Rationale: CHAR(40) is not RDBMS-specific + history_type VARCHAR(63) NOT NULL, + last_ido_id BIGINT NOT NULL, - CONSTRAINT pk_ido_migration_progress PRIMARY KEY (history_type) + CONSTRAINT pk_ido_migration_progress PRIMARY KEY (environment_id, history_type) ); diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index f8de3848..8b96bbdd 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -86,7 +86,7 @@ func main() { // computeProgress figures out which data has already been migrated // not to start from the beginning every time in the following migrate(). - computeProgress(idb) + computeProgress(idb, envId) // On rationale read buildEventTimeCache() and buildPreviousHardStateCache() docs. log.Info("Filling cache") @@ -207,15 +207,18 @@ var idoMigrationProgressSchema string // computeProgress initializes types[*].lastId, types[*].total and types[*].done. // (On non-recoverable errors the whole program exits.) -func computeProgress(idb *icingadb.DB) { +func computeProgress(idb *icingadb.DB, envId []byte) { if _, err := idb.Exec(idoMigrationProgressSchema); err != nil { log.Fatalf("%+v", errors.Wrap(err, "can't create table ido_migration_progress")) } + envIdHex := hex.EncodeToString(envId) types.forEach(func(ht *historyType) { - var query = idb.Rebind("SELECT last_ido_id FROM ido_migration_progress WHERE history_type=?") + var query = idb.Rebind( + "SELECT last_ido_id FROM ido_migration_progress WHERE environment_id=? AND history_type=?", + ) - switch err := idb.Get(&ht.lastId, query, ht.name); err { + switch err := idb.Get(&ht.lastId, query, envIdHex, ht.name); err { case nil, sql.ErrNoRows: default: log.With("backend", "Icinga DB", "query", query, "args", []interface{}{ht.name}). @@ -341,6 +344,7 @@ func migrateOneType[IdoRow any]( } upsertProgress, _ := idb.BuildUpsertStmt(&IdoMigrationProgress{}) + envIdHex := hex.EncodeToString(envId) ht.bar.SetCurrent(ht.done) @@ -401,7 +405,7 @@ func migrateOneType[IdoRow any]( args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} _, err := tx.NamedExec(upsertProgress, &IdoMigrationProgress{ - IdoMigrationProgressUpserter{lastIdoId}, ht.name, + IdoMigrationProgressUpserter{lastIdoId}, envIdHex, ht.name, }) if err != nil { log.With("backend", "Icinga DB", "dml", upsertProgress, "args", args). diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 6717c106..96c8552b 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -77,6 +77,7 @@ func (impu *IdoMigrationProgressUpserter) Upsert() interface{} { type IdoMigrationProgress struct { IdoMigrationProgressUpserter `json:",inline"` + EnvironmentId string `json:"environment_id"` HistoryType string `json:"history_type"` } From 1c381cfef82e09e815a79603d47a5c3badc85284 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 28 Jul 2022 19:00:18 +0200 Subject: [PATCH 082/103] cmd/ido2icingadb: make cache filling fast again by COMMITing less frequently. --- cmd/ido2icingadb/cache.go | 2 +- cmd/ido2icingadb/misc.go | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index b2a9e1b4..2a829841 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -247,7 +247,7 @@ func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, commitPeriodically func( do(&tx, func() { // commitPeriodically callsSinceLastTx++ - if callsSinceLastTx == bulk { + if callsSinceLastTx == 10000 { if err := tx.Commit(); err != nil { logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 96c8552b..ff9cfeeb 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -87,8 +87,6 @@ var ( _ contracts.Upserter = (*IdoMigrationProgress)(nil) ) -const bulk = 100 - // log is the root logger. var log = func() *zap.SugaredLogger { logger, err := zap.NewDevelopmentConfig().Build() @@ -154,7 +152,7 @@ func sliceIdoHistory[Row any]( } args["checkpoint"] = checkpoint - args["bulk"] = bulk + args["bulk"] = 100 if snapshot.DriverName() != driver.MySQL { query = strings.ReplaceAll(query, " USE INDEX (PRIMARY)", "") From 36d07aa7cdae1d17c89b18e8f59fa7a8752431c7 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Jul 2022 19:04:02 +0200 Subject: [PATCH 083/103] cmd/ido2icingadb: add docs --- cmd/ido2icingadb/README.md | 144 +++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 cmd/ido2icingadb/README.md diff --git a/cmd/ido2icingadb/README.md b/cmd/ido2icingadb/README.md new file mode 100644 index 00000000..cc2c6d87 --- /dev/null +++ b/cmd/ido2icingadb/README.md @@ -0,0 +1,144 @@ +# IDO2IcingaDB + +![Icinga Logo] + +#### Table of Contents + +- [About](#about) +- [License](#license) +- [Installation](#installation) +- [Usage](#usage) +- [Support](#support) +- [Contributing](#contributing) + +## About + +This commandline tool migrates history data from [IDO] to [Icinga DB]. +Or, more precisely: from the IDO SQL database to the Icinga DB one. + +!!! info + + Everything else is already populated by Icinga DB itself. + Only the past history data of existing IDO setups + isn't known to Icinga DB without migration from IDO. + +## License + +This tool and its documentation are licensed under the terms +of the GNU General Public License Version 2, +you will find a copy of this license in the [LICENSE] file. + +## Installation + +1. Add the official release repository as described + in the [installation chapter] of the Icinga DB documentation. +2. Install the package `ido2icingadb`. + +## Usage + +### Icinga DB + +1. Make sure Icinga DB is up, running and writing to its database. +2. Optionally disable Icinga 2's IDO feature. + +!!! warning + + Migration will cause duplicate Icinga DB events + for the period both IDO and Icinga DB are active. + Disable the IDO feature -- the sooner, the better! + Or read on while not disabling it yet. + There is a way to avoid duplicate events. + +### Configuration file + +Create a YAML file like this somewhere: + +```yaml +icinga2: + # Content of /var/lib/icinga2/icingadb.env + env: "da39a3ee5e6b4b0d3255bfef95601890afBADHEX" + # Name of the main Icinga 2 endpoint writing to IDO + endpoint: master-1 +# IDO database +ido: + type: pgsql + host: 192.0.2.1 + port: 5432 + database: icinga + user: icinga + password: CHANGEME +# Icinga DB database +icingadb: + type: mysql + host: 2001:db8::1 + port: 3306 + database: icingadb + user: icingadb + password: CHANGEME +``` + +### Cache directory + +Choose a (not necessarily yet existing) directory for IDO2IcingaDB's +internal cache. If either there isn't much to migrate or the migration +process won't be interrupted by a reboot (of the machine +IDO2IcingaDB/database runs on), `mktemp -d` is enough. + +### Actual migration + +Run: + +```shell +ido2icingadb -c ido2icingadb.yml -t ~/ido2icingadb.tmp +``` + +In case of an interrupt re-run. + +!!! tip + + If there is much to migrate, use e.g. tmux to + protect yourself against SSH connection losses. + +### Avoid duplicate events (optional) + +The easiest option is to both enable Icinga DB +and disable IDO within one Icinga 2 reload. + +But if this doesn't work on first try, you'll lose history. +Not to lose any data consider everything done above just a test. +Reset Icinga DB and re-migrate as follows: + +!!! warning + + The following assumes you didn't disable the IDO, yet! + +1. Disable the Icinga DB feature: `icinga2 feature disable icingadb` +2. Reload Icinga 2: `systemctl reload icinga2` +3. Stop Icinga DB: `systemctl stop icingadb` +4. Reset Redis: `icingadb-redis-cli flushdb` +5. Re-create Icinga DB's database + (drop database, create database, import schema, grant permissions) +6. Start Icinga DB: `systemctl start icingadb` +7. Both re-enable Icinga DB and disable IDO within one Icinga 2 reload +8. Migrate history again, with a clean cache directory (just to be sure) + +## Support + +Check the [project website] for status updates. Join the [community channels] +for questions or ask an Icinga partner for [professional support]. + +## Contributing + +There are many ways to contribute to Icinga -- whether it be sending patches, +testing, reporting bugs, or reviewing and updating the documentation. Every +contribution is appreciated! + + +[Icinga Logo]: https://icinga.com/wp-content/uploads/2014/06/icinga_logo.png +[IDO]: https://icinga.com/docs/icinga-2/latest/doc/14-features/#ido-database-db-ido +[Icinga DB]: https://icinga.com/docs/icinga-db/latest/doc/01-About/ +[LICENSE]: ./LICENSE +[installation chapter]: https://icinga.com/docs/icingadb/latest/doc/02-Installation/ +[project website]: https://icinga.com +[community channels]: https://icinga.com/community/ +[professional support]: https://icinga.com/support/ From 8bf8a6fe75b07947a122dc6311c1679134d4f709 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 3 Aug 2022 11:23:29 +0200 Subject: [PATCH 084/103] cmd/ido2icingadb: rename to Icinga DB Migration --- cmd/ido2icingadb/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/ido2icingadb/README.md b/cmd/ido2icingadb/README.md index cc2c6d87..d8b715a4 100644 --- a/cmd/ido2icingadb/README.md +++ b/cmd/ido2icingadb/README.md @@ -1,4 +1,4 @@ -# IDO2IcingaDB +# Icinga DB Migration ![Icinga Logo] @@ -32,7 +32,7 @@ you will find a copy of this license in the [LICENSE] file. 1. Add the official release repository as described in the [installation chapter] of the Icinga DB documentation. -2. Install the package `ido2icingadb`. +2. Install the package `icingadb-migration`. ## Usage @@ -79,17 +79,17 @@ icingadb: ### Cache directory -Choose a (not necessarily yet existing) directory for IDO2IcingaDB's +Choose a (not necessarily yet existing) directory for Icinga DB Migration's internal cache. If either there isn't much to migrate or the migration process won't be interrupted by a reboot (of the machine -IDO2IcingaDB/database runs on), `mktemp -d` is enough. +Icinga DB migration/database runs on), `mktemp -d` is enough. ### Actual migration Run: ```shell -ido2icingadb -c ido2icingadb.yml -t ~/ido2icingadb.tmp +icingadb-migrate -c icingadb-migration.yml -t ~/icingadb-migration.tmp ``` In case of an interrupt re-run. From aa571f08567d4dfaaaff5a3ffc36e65b97180fd7 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 5 Aug 2022 11:21:31 +0200 Subject: [PATCH 085/103] cmd/ido2icingadb: remove unnecessary mutex for the sake of speed. --- cmd/ido2icingadb/main.go | 14 ++------------ cmd/ido2icingadb/misc.go | 24 +++++++++++------------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 8b96bbdd..4ec3705b 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -11,7 +11,6 @@ import ( "github.com/goccy/go-yaml" "github.com/icinga/icingadb/pkg/com" "github.com/icinga/icingadb/pkg/config" - "github.com/icinga/icingadb/pkg/driver" "github.com/icinga/icingadb/pkg/icingadb" "github.com/icinga/icingadb/pkg/logging" icingadbTypes "github.com/icinga/icingadb/pkg/types" @@ -25,7 +24,6 @@ import ( "os" "path" "regexp" - "sync" "time" ) @@ -279,7 +277,6 @@ func fillCache() { // migrate does the actual migration. func migrate(c *Config, idb *icingadb.DB, envId []byte) { endpointId := sha1.Sum([]byte(c.Icinga2.Endpoint)) - idbTx := &sync.Mutex{} progress := mpb.New() for _, ht := range types { @@ -287,7 +284,7 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { } types.forEach(func(ht *historyType) { - ht.migrate(c, idb, envId, endpointId, idbTx, ht) + ht.migrate(c, idb, envId, endpointId, ht) }) progress.Wait() @@ -295,7 +292,7 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { // migrate does the actual migration for one history type. func migrateOneType[IdoRow any]( - c *Config, idb *icingadb.DB, envId []byte, endpointId [sha1.Size]byte, idbTx *sync.Mutex, ht *historyType, + c *Config, idb *icingadb.DB, envId []byte, endpointId [sha1.Size]byte, ht *historyType, convertRows func(env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []IdoRow) (icingaDbInserts, icingaDbUpserts [][]any, checkpoint any), @@ -359,13 +356,6 @@ func migrateOneType[IdoRow any]( // ... and insert them: - if idb.DriverName() == driver.MySQL { - // Avoid MySQL error 1205 (Lock wait timeout exceeded; try restarting transaction) - // due to concurrent transactions upsert the same table (history). - idbTx.Lock() - defer idbTx.Unlock() - } - tx, err := idb.Beginx() if err != nil { log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index ff9cfeeb..19e17ba4 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -15,7 +15,6 @@ import ( "go.uber.org/zap" "golang.org/x/sync/errgroup" "strings" - "sync" "time" ) @@ -201,8 +200,7 @@ type historyType struct { // migrationQuery SELECTs source data for actual migration. migrationQuery string // migrate does the actual migration. - migrate func(c *Config, idb *icingadb.DB, envId []byte, - endpointId [sha1.Size]byte, idbTx *sync.Mutex, ht *historyType) + migrate func(c *Config, idb *icingadb.DB, envId []byte, endpointId [sha1.Size]byte, ht *historyType) // cacheFile locates .sqlite3. cacheFile string @@ -259,8 +257,8 @@ var types = historyTypes{ idoTable: "icinga_commenthistory", idoIdColumn: "commenthistory_id", migrationQuery: commentMigrationQuery, - migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { - migrateOneType(c, idb, envId, endpId, idbTx, ht, convertCommentRows) + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, ht *historyType) { + migrateOneType(c, idb, envId, endpId, ht, convertCommentRows) }, }, { @@ -268,8 +266,8 @@ var types = historyTypes{ idoTable: "icinga_downtimehistory", idoIdColumn: "downtimehistory_id", migrationQuery: downtimeMigrationQuery, - migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { - migrateOneType(c, idb, envId, endpId, idbTx, ht, convertDowntimeRows) + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, ht *historyType) { + migrateOneType(c, idb, envId, endpId, ht, convertDowntimeRows) }, }, { @@ -284,8 +282,8 @@ var types = historyTypes{ }) }, migrationQuery: flappingMigrationQuery, - migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { - migrateOneType(c, idb, envId, endpId, idbTx, ht, convertFlappingRows) + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, ht *historyType) { + migrateOneType(c, idb, envId, endpId, ht, convertFlappingRows) }, }, { @@ -300,8 +298,8 @@ var types = historyTypes{ }, cacheLimitQuery: "SELECT MAX(history_id) FROM previous_hard_state", migrationQuery: notificationMigrationQuery, - migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { - migrateOneType(c, idb, envId, endpId, idbTx, ht, convertNotificationRows) + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, ht *historyType) { + migrateOneType(c, idb, envId, endpId, ht, convertNotificationRows) }, }, { @@ -314,8 +312,8 @@ var types = historyTypes{ }, cacheLimitQuery: "SELECT MAX(history_id) FROM previous_hard_state", migrationQuery: stateMigrationQuery, - migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, idbTx *sync.Mutex, ht *historyType) { - migrateOneType(c, idb, envId, endpId, idbTx, ht, convertStateRows) + migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, ht *historyType) { + migrateOneType(c, idb, envId, endpId, ht, convertStateRows) }, }, } From 690fcfa190fea2c388fe8ff480c806a737b15cfa Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 5 Aug 2022 11:23:07 +0200 Subject: [PATCH 086/103] cmd/ido2icingadb: chunkCacheTx(): commit less often for the sake of speed. --- cmd/ido2icingadb/cache.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 2a829841..71e9fd17 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "math" "strings" + "time" ) //go:embed embed/event_time_cache_schema.sql @@ -238,16 +239,17 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { // (On non-recoverable errors the whole program exits.) func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, commitPeriodically func())) { logger := log.With("backend", "cache") - var callsSinceLastTx int tx, err := cache.Beginx() if err != nil { logger.Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) } + const commitInterval = 5 * time.Minute + nextCommit := time.Now().Add(commitInterval) + do(&tx, func() { // commitPeriodically - callsSinceLastTx++ - if callsSinceLastTx == 10000 { + if now := time.Now(); now.After(nextCommit) { if err := tx.Commit(); err != nil { logger.Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) } @@ -259,7 +261,7 @@ func chunkCacheTx(cache *sqlx.DB, do func(tx **sqlx.Tx, commitPeriodically func( logger.Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) } - callsSinceLastTx = 0 + nextCommit = nextCommit.Add(commitInterval) } }) From 23130d7be8d4e9205257aa375a9839887480ce49 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 11 Aug 2022 13:48:38 +0200 Subject: [PATCH 087/103] cmd/ido2icingadb: remove unnecessary transactions for the sake of speed. --- cmd/ido2icingadb/main.go | 13 ++----------- cmd/ido2icingadb/misc.go | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 4ec3705b..8e49dbb6 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -356,11 +356,6 @@ func migrateOneType[IdoRow any]( // ... and insert them: - tx, err := idb.Beginx() - if err != nil { - log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't begin transaction")) - } - for _, op := range []struct { data [][]any stmtBuilder func(any) (string, int) @@ -383,7 +378,7 @@ func migrateOneType[IdoRow any]( context.Background(), ch, idb.BatchSizeByPlaceholders(placeholders), com.NeverSplit[any], ) for rows := range bulk { - if _, err := tx.NamedExec(query, rows); err != nil { + if _, err := idb.NamedExec(query, rows); err != nil { log.With("backend", "Icinga DB", "dml", query, "args", rows). Fatalf("%+v", errors.Wrap(err, "can't perform DML")) } @@ -394,7 +389,7 @@ func migrateOneType[IdoRow any]( if lastIdoId != nil { args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} - _, err := tx.NamedExec(upsertProgress, &IdoMigrationProgress{ + _, err := idb.NamedExec(upsertProgress, &IdoMigrationProgress{ IdoMigrationProgressUpserter{lastIdoId}, envIdHex, ht.name, }) if err != nil { @@ -403,10 +398,6 @@ func migrateOneType[IdoRow any]( } } - if err := tx.Commit(); err != nil { - log.With("backend", "Icinga DB").Fatalf("%+v", errors.Wrap(err, "can't commit transaction")) - } - ht.bar.IncrBy(len(idoRows)) return lastIdoId }, diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 19e17ba4..0738d61d 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -151,7 +151,7 @@ func sliceIdoHistory[Row any]( } args["checkpoint"] = checkpoint - args["bulk"] = 100 + args["bulk"] = 10000 if snapshot.DriverName() != driver.MySQL { query = strings.ReplaceAll(query, " USE INDEX (PRIMARY)", "") From adcd004231af108519038eca160ecee28b3507ea Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 19 Aug 2022 15:24:23 +0200 Subject: [PATCH 088/103] Introduce DB#CreateIgnoreStreamed() --- pkg/icingadb/db.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pkg/icingadb/db.go b/pkg/icingadb/db.go index bbd02e6d..4494eaf4 100644 --- a/pkg/icingadb/db.go +++ b/pkg/icingadb/db.go @@ -529,6 +529,28 @@ func (db *DB) CreateStreamed( ) } +// CreateIgnoreStreamed bulk creates the specified entities via NamedBulkExec. +// The insert statement is created using BuildInsertIgnoreStmt with the first entity from the entities stream. +// Bulk size is controlled via Options.MaxPlaceholdersPerStatement and +// concurrency is controlled via Options.MaxConnectionsPerTable. +// Entities for which the query ran successfully will be passed to onSuccess. +func (db *DB) CreateIgnoreStreamed( + ctx context.Context, entities <-chan contracts.Entity, onSuccess ...OnSuccess[contracts.Entity], +) error { + first, forward, err := com.CopyFirst(ctx, entities) + if first == nil { + return errors.Wrap(err, "can't copy first entity") + } + + sem := db.GetSemaphoreForTable(utils.TableName(first)) + stmt, placeholders := db.BuildInsertIgnoreStmt(first) + + return db.NamedBulkExec( + ctx, stmt, db.BatchSizeByPlaceholders(placeholders), sem, + forward, com.SplitOnDupId[contracts.Entity], onSuccess..., + ) +} + // UpsertStreamed bulk upserts the specified entities via NamedBulkExec. // The upsert statement is created using BuildUpsertStmt with the first entity from the entities stream. // Bulk size is controlled via Options.MaxPlaceholdersPerStatement and From 6804bbdb54eab4bf475dd878a6c236e0c2827cee Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 19 Aug 2022 15:46:46 +0200 Subject: [PATCH 089/103] cmd/ido2icingadb: increase parallelism --- cmd/ido2icingadb/convert.go | 32 ++++++++++++++++---------------- cmd/ido2icingadb/main.go | 30 +++++++++++++----------------- cmd/ido2icingadb/misc.go | 2 +- 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index df318e08..d858033d 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -50,8 +50,8 @@ type commentRow = struct { func convertCommentRows( env string, envId, endpointId icingadbTypes.Binary, _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []commentRow, -) (icingaDbInserts, _ [][]any, checkpoint any) { - var commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck []any +) (icingaDbInserts, _ [][]contracts.Entity, checkpoint any) { + var commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck []contracts.Entity for _, row := range idoRows { typ := objectTypes[row.ObjecttypeId] @@ -213,7 +213,7 @@ func convertCommentRows( checkpoint = row.CommenthistoryId } - icingaDbInserts = [][]any{commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck} + icingaDbInserts = [][]contracts.Entity{commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck} return } @@ -242,8 +242,8 @@ type downtimeRow = struct { func convertDowntimeRows( env string, envId, endpointId icingadbTypes.Binary, _ func(interface{}, string, ...interface{}), _ *sqlx.Tx, idoRows []downtimeRow, -) (icingaDbInserts, _ [][]any, checkpoint any) { - var downtimeHistory, allHistory, sla []interface{} +) (icingaDbInserts, _ [][]contracts.Entity, checkpoint any) { + var downtimeHistory, allHistory, sla []contracts.Entity for _, row := range idoRows { id := calcObjectId(env, row.Name) @@ -367,7 +367,7 @@ func convertDowntimeRows( checkpoint = row.DowntimehistoryId } - icingaDbInserts = [][]interface{}{downtimeHistory, allHistory, sla} + icingaDbInserts = [][]contracts.Entity{downtimeHistory, allHistory, sla} return } @@ -402,7 +402,7 @@ type flappingRow = struct { func convertFlappingRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []flappingRow, -) (icingaDbInserts, icingaDbUpserts [][]any, checkpoint any) { +) (icingaDbInserts, icingaDbUpserts [][]contracts.Entity, checkpoint any) { if len(idoRows) < 1 { return } @@ -423,7 +423,7 @@ func convertFlappingRows( cachedById[c.HistoryId] = convertTime(c.EventTime, c.EventTimeUsec) } - var flappingHistory, flappingHistoryUpserts, allHistory []any + var flappingHistory, flappingHistoryUpserts, allHistory []contracts.Entity for _, row := range idoRows { ts := convertTime(row.EventTime, row.EventTimeUsec) @@ -535,8 +535,8 @@ func convertFlappingRows( checkpoint = row.FlappinghistoryId } - icingaDbInserts = [][]interface{}{flappingHistory, allHistory} - icingaDbUpserts = [][]interface{}{flappingHistoryUpserts} + icingaDbInserts = [][]contracts.Entity{flappingHistory, allHistory} + icingaDbUpserts = [][]contracts.Entity{flappingHistoryUpserts} return } @@ -557,7 +557,7 @@ type notificationRow = struct { func convertNotificationRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, idoRows []notificationRow, -) (icingaDbInserts, _ [][]any, checkpoint any) { +) (icingaDbInserts, _ [][]contracts.Entity, checkpoint any) { if len(idoRows) < 1 { return } @@ -604,7 +604,7 @@ func convertNotificationRows( perId[contact.Name1] = struct{}{} } - var notificationHistory, userNotificationHistory, allHistory []interface{} + var notificationHistory, userNotificationHistory, allHistory []contracts.Entity for _, row := range idoRows { previousHardState, ok := cachedById[row.NotificationId] if !ok { @@ -696,7 +696,7 @@ func convertNotificationRows( checkpoint = row.NotificationId } - icingaDbInserts = [][]interface{}{notificationHistory, userNotificationHistory, allHistory} + icingaDbInserts = [][]contracts.Entity{notificationHistory, userNotificationHistory, allHistory} return } @@ -752,7 +752,7 @@ type stateRow = struct { func convertStateRows( env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), _ *sqlx.Tx, idoRows []stateRow, -) (icingaDbInserts, _ [][]any, checkpoint any) { +) (icingaDbInserts, _ [][]contracts.Entity, checkpoint any) { if len(idoRows) < 1 { return } @@ -771,7 +771,7 @@ func convertStateRows( cachedById[c.HistoryId] = c.PreviousHardState } - var stateHistory, allHistory, sla []interface{} + var stateHistory, allHistory, sla []contracts.Entity for _, row := range idoRows { previousHardState, ok := cachedById[row.StatehistoryId] if !ok { @@ -853,6 +853,6 @@ func convertStateRows( checkpoint = row.StatehistoryId } - icingaDbInserts = [][]interface{}{stateHistory, allHistory, sla} + icingaDbInserts = [][]contracts.Entity{stateHistory, allHistory, sla} return } diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 8e49dbb6..26776844 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -9,17 +9,19 @@ import ( "fmt" "github.com/creasty/defaults" "github.com/goccy/go-yaml" - "github.com/icinga/icingadb/pkg/com" "github.com/icinga/icingadb/pkg/config" + "github.com/icinga/icingadb/pkg/contracts" "github.com/icinga/icingadb/pkg/icingadb" "github.com/icinga/icingadb/pkg/logging" icingadbTypes "github.com/icinga/icingadb/pkg/types" + "github.com/icinga/icingadb/pkg/utils" "github.com/jessevdk/go-flags" "github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx/reflectx" _ "github.com/mattn/go-sqlite3" "github.com/pkg/errors" "github.com/vbauerster/mpb/v6" + "go.uber.org/zap" "golang.org/x/sync/errgroup" "os" "path" @@ -175,7 +177,7 @@ func connectAll(c *Config) (ido, idb *icingadb.DB) { // connect connects to which DB as cfg specifies. (On non-recoverable errors the whole program exits.) func connect(which string, cfg *config.Database) *icingadb.DB { - db, err := cfg.Open(logging.NewLogger(log, 20*time.Second)) + db, err := cfg.Open(logging.NewLogger(zap.NewNop().Sugar(), 20*time.Second)) if err != nil { log.With("backend", which).Fatalf("%+v", errors.Wrap(err, "can't connect to database")) } @@ -295,7 +297,7 @@ func migrateOneType[IdoRow any]( c *Config, idb *icingadb.DB, envId []byte, endpointId [sha1.Size]byte, ht *historyType, convertRows func(env string, envId, endpointId icingadbTypes.Binary, selectCache func(dest interface{}, query string, args ...interface{}), ido *sqlx.Tx, - idoRows []IdoRow) (icingaDbInserts, icingaDbUpserts [][]any, checkpoint any), + idoRows []IdoRow) (icingaDbInserts, icingaDbUpserts [][]contracts.Entity, checkpoint any), ) { var lastQuery string var lastStmt *sqlx.Stmt @@ -357,31 +359,25 @@ func migrateOneType[IdoRow any]( // ... and insert them: for _, op := range []struct { - data [][]any - stmtBuilder func(any) (string, int) - }{{inserts, idb.BuildInsertIgnoreStmt}, {upserts, idb.BuildUpsertStmt}} { + kind string + data [][]contracts.Entity + streamer func(context.Context, <-chan contracts.Entity, ...icingadb.OnSuccess[contracts.Entity]) error + }{{"INSERT IGNORE", inserts, idb.CreateIgnoreStreamed}, {"UPSERT", upserts, idb.UpsertStreamed}} { for _, table := range op.data { if len(table) < 1 { continue } - query, placeholders := op.stmtBuilder(table[0]) - - ch := make(chan any, len(table)) + ch := make(chan contracts.Entity, len(table)) for _, row := range table { ch <- row } close(ch) - bulk := com.Bulk( - context.Background(), ch, idb.BatchSizeByPlaceholders(placeholders), com.NeverSplit[any], - ) - for rows := range bulk { - if _, err := idb.NamedExec(query, rows); err != nil { - log.With("backend", "Icinga DB", "dml", query, "args", rows). - Fatalf("%+v", errors.Wrap(err, "can't perform DML")) - } + if err := op.streamer(context.Background(), ch); err != nil { + log.With("backend", "Icinga DB", "op", op.kind, "table", utils.TableName(table[0])). + Fatalf("%+v", errors.Wrap(err, "can't perform DML")) } } } diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 0738d61d..f47ba64d 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -151,7 +151,7 @@ func sliceIdoHistory[Row any]( } args["checkpoint"] = checkpoint - args["bulk"] = 10000 + args["bulk"] = 20000 if snapshot.DriverName() != driver.MySQL { query = strings.ReplaceAll(query, " USE INDEX (PRIMARY)", "") From 6794252ba1f5aa08dbf32d97543b8cc841663f81 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 19 Aug 2022 15:53:35 +0200 Subject: [PATCH 090/103] cmd/ido2icingadb: remove unused code --- cmd/ido2icingadb/convert.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index d858033d..3074e5d9 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -371,21 +371,6 @@ func convertDowntimeRows( return } -// FlappingEnd updates an already migrated start event with the end event info. -type FlappingEnd struct { - Id icingadbTypes.Binary `json:"id"` - EndTime icingadbTypes.UnixMilli `json:"end_time"` - PercentStateChangeEnd icingadbTypes.Float `json:"percent_state_change_end"` -} - -// Assert interface compliance. -var _ contracts.TableNamer = (*FlappingEnd)(nil) - -// TableName implements the contracts.TableNamer interface. -func (*FlappingEnd) TableName() string { - return "flapping_history" -} - type flappingRow = struct { FlappinghistoryId uint64 EventTime int64 From 8402eb266f8024b01b31eb833a1efa50040f018f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 19 Aug 2022 16:39:03 +0200 Subject: [PATCH 091/103] cmd/ido2icingadb: show ops/s --- cmd/ido2icingadb/misc.go | 92 +++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 25 deletions(-) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index f47ba64d..a4e59368 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -3,6 +3,7 @@ package main import ( "context" "crypto/sha1" + "fmt" "github.com/icinga/icingadb/pkg/contracts" "github.com/icinga/icingadb/pkg/driver" "github.com/icinga/icingadb/pkg/icingadb" @@ -18,8 +19,9 @@ import ( "time" ) -// eta indicates the ETA for possibly slowly incrementing progresses possibly starting from >0%. -type eta struct { +// slowlyIncrementingProgressDecorator is the base decorator +// for possibly slowly incrementing progresses possibly starting from >0%. +type slowlyIncrementingProgressDecorator struct { decor.WC // startProgress is the first progress >0 seen by Decor. @@ -32,38 +34,73 @@ type eta struct { lastTime time.Time } +// update is to be called by Decor and returns whether sipd is warmed up. +func (sipd *slowlyIncrementingProgressDecorator) update(s decor.Statistics) bool { + if s.Completed || s.Current < 1 { + return false + } + + if sipd.startProgress < 1 { + sipd.startProgress = s.Current + sipd.startTime = time.Now() + sipd.lastProgress = sipd.startProgress + sipd.lastTime = sipd.startTime + + return false + } + + if s.Current == sipd.startProgress { + return false + } + + if s.Current > sipd.lastProgress { + sipd.lastProgress = s.Current + sipd.lastTime = time.Now() + } + + return true +} + +// eta indicates the ETA for possibly slowly incrementing progresses possibly starting from >0%. +type eta struct { + slowlyIncrementingProgressDecorator +} + // Decor implements the decor.Decorator interface. func (e *eta) Decor(s decor.Statistics) string { - if s.Completed || s.Current < 1 { + if e.update(s) { + timePerItem := float64(e.lastTime.Sub(e.startTime)) / float64(e.lastProgress-e.startProgress) + lastETA := time.Duration(float64(s.Total-s.Current) * timePerItem) + + return e.FormatMsg(((lastETA - time.Since(e.lastTime)) / time.Second * time.Second).String()) + } else { return "" } +} - if e.startProgress < 1 { - e.startProgress = s.Current - e.startTime = time.Now() - e.lastProgress = e.startProgress - e.lastTime = e.startTime +// eta indicates ops/s for possibly slowly incrementing progresses possibly starting from >0%. +type opsPerSec struct { + slowlyIncrementingProgressDecorator +} +// Decor implements the decor.Decorator interface. +func (ops *opsPerSec) Decor(s decor.Statistics) string { + if ops.update(s) { + return ops.FormatMsg(fmt.Sprintf( + "%.0f/s", + float64(ops.lastProgress-ops.startProgress)/ + (float64(ops.lastTime.Sub(ops.startTime))/float64(time.Second)), + )) + } else { return "" } - - if s.Current == e.startProgress { - return "" - } - - if s.Current > e.lastProgress { - e.lastProgress = s.Current - e.lastTime = time.Now() - } - - timePerItem := float64(e.lastTime.Sub(e.startTime)) / float64(e.lastProgress-e.startProgress) - lastETA := time.Duration(float64(s.Total-s.Current) * timePerItem) - - return e.FormatMsg(((lastETA - time.Since(e.lastTime)) / time.Second * time.Second).String()) } // Assert interface compliance. -var _ decor.Decorator = (*eta)(nil) +var ( + _ decor.Decorator = (*eta)(nil) + _ decor.Decorator = (*opsPerSec)(nil) +) type IdoMigrationProgressUpserter struct { LastIdoId any `json:"last_ido_id"` @@ -220,9 +257,14 @@ type historyType struct { // setupBar (re-)initializes ht.bar. func (ht *historyType) setupBar(progress *mpb.Progress) { - e := &eta{WC: decor.WC{W: 4}} + e := &eta{} + ops := &opsPerSec{} + + e.W = 4 + ops.W = 4 e.Init() + ops.Init() ht.bar = progress.AddBar( ht.total, @@ -231,7 +273,7 @@ func (ht *historyType) setupBar(progress *mpb.Progress) { decor.Name(ht.name, decor.WC{W: len(ht.name) + 1, C: decor.DidentRight}), decor.Percentage(decor.WC{W: 5}), ), - mpb.AppendDecorators(e), + mpb.AppendDecorators(e, decor.Name(" "), ops), ) } From dbf394fb0df117ed3acbfd0a957896285ad90ab8 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 24 Aug 2022 16:26:18 +0200 Subject: [PATCH 092/103] cmd/ido2icingadb: make sure not to return false-positive nil (i.e. EOF) checkpoints --- cmd/ido2icingadb/convert.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/ido2icingadb/convert.go b/cmd/ido2icingadb/convert.go index 3074e5d9..8f4c47df 100644 --- a/cmd/ido2icingadb/convert.go +++ b/cmd/ido2icingadb/convert.go @@ -54,6 +54,8 @@ func convertCommentRows( var commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck []contracts.Entity for _, row := range idoRows { + checkpoint = row.CommenthistoryId + typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) serviceId := calcServiceId(env, row.Name1, row.Name2) @@ -209,8 +211,6 @@ func convertCommentRows( allHistoryAck = append(allHistoryAck, h2) } } - - checkpoint = row.CommenthistoryId } icingaDbInserts = [][]contracts.Entity{commentHistory, acknowledgementHistory, allHistoryComment, allHistoryAck} @@ -246,6 +246,8 @@ func convertDowntimeRows( var downtimeHistory, allHistory, sla []contracts.Entity for _, row := range idoRows { + checkpoint = row.DowntimehistoryId + id := calcObjectId(env, row.Name) typ := objectTypes[row.ObjecttypeId] hostId := calcObjectId(env, row.Name1) @@ -363,8 +365,6 @@ func convertDowntimeRows( s.DowntimeEnd.History = s sla = append(sla, s) - - checkpoint = row.DowntimehistoryId } icingaDbInserts = [][]contracts.Entity{downtimeHistory, allHistory, sla} @@ -410,6 +410,8 @@ func convertFlappingRows( var flappingHistory, flappingHistoryUpserts, allHistory []contracts.Entity for _, row := range idoRows { + checkpoint = row.FlappinghistoryId + ts := convertTime(row.EventTime, row.EventTimeUsec) // Needed for ID (see below). @@ -516,8 +518,6 @@ func convertFlappingRows( h.EventTime.History = h allHistory = append(allHistory, h) } - - checkpoint = row.FlappinghistoryId } icingaDbInserts = [][]contracts.Entity{flappingHistory, allHistory} @@ -591,6 +591,8 @@ func convertNotificationRows( var notificationHistory, userNotificationHistory, allHistory []contracts.Entity for _, row := range idoRows { + checkpoint = row.NotificationId + previousHardState, ok := cachedById[row.NotificationId] if !ok { continue @@ -677,8 +679,6 @@ func convertNotificationRows( UserId: userId, }) } - - checkpoint = row.NotificationId } icingaDbInserts = [][]contracts.Entity{notificationHistory, userNotificationHistory, allHistory} @@ -758,6 +758,8 @@ func convertStateRows( var stateHistory, allHistory, sla []contracts.Entity for _, row := range idoRows { + checkpoint = row.StatehistoryId + previousHardState, ok := cachedById[row.StatehistoryId] if !ok { continue @@ -834,8 +836,6 @@ func convertStateRows( PreviousHardState: previousHardState, }) } - - checkpoint = row.StatehistoryId } icingaDbInserts = [][]contracts.Entity{stateHistory, allHistory, sla} From 2c0e927a49ea5ef60d05d831e17d73b493a4c921 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 6 Sep 2022 15:50:04 +0200 Subject: [PATCH 093/103] cmd/ido2icingadb: allow custom input data time range --- cmd/ido2icingadb/README.md | 26 ++++++- cmd/ido2icingadb/cache.go | 6 +- cmd/ido2icingadb/embed/comment_query.sql | 3 +- cmd/ido2icingadb/embed/downtime_query.sql | 3 +- cmd/ido2icingadb/embed/flapping_query.sql | 3 +- cmd/ido2icingadb/embed/notification_query.sql | 3 +- cmd/ido2icingadb/embed/state_query.sql | 3 +- cmd/ido2icingadb/main.go | 63 +++++++++++++++-- cmd/ido2icingadb/misc.go | 68 ++++++++++++------- 9 files changed, 138 insertions(+), 40 deletions(-) diff --git a/cmd/ido2icingadb/README.md b/cmd/ido2icingadb/README.md index d8b715a4..395822ec 100644 --- a/cmd/ido2icingadb/README.md +++ b/cmd/ido2icingadb/README.md @@ -47,7 +47,7 @@ you will find a copy of this license in the [LICENSE] file. for the period both IDO and Icinga DB are active. Disable the IDO feature -- the sooner, the better! Or read on while not disabling it yet. - There is a way to avoid duplicate events. + There are ways to avoid duplicate events. ### Configuration file @@ -67,6 +67,9 @@ ido: database: icinga user: icinga password: CHANGEME + # Input time range + #from: 0 + #to: 2147483647 # Icinga DB database icingadb: type: mysql @@ -77,6 +80,21 @@ icingadb: password: CHANGEME ``` +#### Input time range + +By default, everything is migrated. If you wish, you can restrict the input +data's start and/or end by giving `from` and/or `to` under `ido:` as Unix +timestamps (in seconds). + +Examples: + +* Now: Run in a shell: `date +%s` +* One year ago: Run in a shell: `date -d -1year +%s` +* Icinga DB usage start time: Query the Icinga DB database: + `SELECT MIN(event_time)/1000 FROM history;` + +The latter is useful for the range end to avoid duplicate events. + ### Cache directory Choose a (not necessarily yet existing) directory for Icinga DB Migration's @@ -101,10 +119,12 @@ In case of an interrupt re-run. ### Avoid duplicate events (optional) -The easiest option is to both enable Icinga DB +The easiest option has already been mentioned in the _Input time range_ section. + +Another option is to both enable Icinga DB and disable IDO within one Icinga 2 reload. -But if this doesn't work on first try, you'll lose history. +But if the latter doesn't work on first try, you'll lose history. Not to lose any data consider everything done above just a test. Reset Icinga DB and re-migrate as follows: diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 71e9fd17..8aaaec0e 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -44,11 +44,12 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { // Stream source data... sliceIdoHistory( - ht.snapshot, + ht, "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ + ht.idoIdColumn+" BETWEEN :fromid AND :toid AND xh."+ ht.idoIdColumn+" > :checkpoint ORDER BY xh."+ht.idoIdColumn+" LIMIT :bulk", nil, checkpoint.MaxId.Int64, // ... since we were interrupted: func(idoRows []row) (checkpoint interface{}) { @@ -154,11 +155,12 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { // Stream source data... sliceIdoHistory( - ht.snapshot, + ht, "SELECT "+strings.Join(idoColumns, ", ")+" FROM "+ht.idoTable+ // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ + ht.idoIdColumn+" BETWEEN :fromid AND :toid AND xh."+ ht.idoIdColumn+" < :checkpoint ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT :bulk", nil, checkpoint, // ... since we were interrupted: func(idoRows []row) (checkpoint interface{}) { diff --git a/cmd/ido2icingadb/embed/comment_query.sql b/cmd/ido2icingadb/embed/comment_query.sql index 5165af3d..774ccf97 100644 --- a/cmd/ido2icingadb/embed/comment_query.sql +++ b/cmd/ido2icingadb/embed/comment_query.sql @@ -5,6 +5,7 @@ SELECT ch.commenthistory_id, UNIX_TIMESTAMP(ch.entry_time) entry_time, ch.deletion_time_usec, ch.name, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 FROM icinga_commenthistory ch USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=ch.object_id -WHERE ch.commenthistory_id > :checkpoint -- where we were interrupted +WHERE ch.commenthistory_id BETWEEN :fromid AND :toid +AND ch.commenthistory_id > :checkpoint -- where we were interrupted ORDER BY ch.commenthistory_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/downtime_query.sql b/cmd/ido2icingadb/embed/downtime_query.sql index 34b9f0ad..e3d36bf0 100644 --- a/cmd/ido2icingadb/embed/downtime_query.sql +++ b/cmd/ido2icingadb/embed/downtime_query.sql @@ -8,6 +8,7 @@ SELECT dh.downtimehistory_id, UNIX_TIMESTAMP(dh.entry_time) entry_time, dh.autho FROM icinga_downtimehistory dh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=dh.object_id LEFT JOIN icinga_scheduleddowntime sd ON sd.scheduleddowntime_id=dh.triggered_by_id -WHERE dh.downtimehistory_id > :checkpoint -- where we were interrupted +WHERE dh.downtimehistory_id BETWEEN :fromid AND :toid +AND dh.downtimehistory_id > :checkpoint -- where we were interrupted ORDER BY dh.downtimehistory_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/flapping_query.sql b/cmd/ido2icingadb/embed/flapping_query.sql index f6f3055d..5e25bded 100644 --- a/cmd/ido2icingadb/embed/flapping_query.sql +++ b/cmd/ido2icingadb/embed/flapping_query.sql @@ -3,6 +3,7 @@ SELECT fh.flappinghistory_id, UNIX_TIMESTAMP(fh.event_time) event_time, fh.high_threshold, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 FROM icinga_flappinghistory fh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=fh.object_id -WHERE fh.flappinghistory_id > :checkpoint -- where we were interrupted +WHERE fh.flappinghistory_id BETWEEN :fromid AND :toid +AND fh.flappinghistory_id > :checkpoint -- where we were interrupted ORDER BY fh.flappinghistory_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/notification_query.sql b/cmd/ido2icingadb/embed/notification_query.sql index 63e1f720..f963b2e9 100644 --- a/cmd/ido2icingadb/embed/notification_query.sql +++ b/cmd/ido2icingadb/embed/notification_query.sql @@ -3,6 +3,7 @@ SELECT n.notification_id, n.notification_reason, UNIX_TIMESTAMP(n.end_time) end_ n.contacts_notified, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 FROM icinga_notifications n USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=n.object_id -WHERE n.notification_id <= :cache_limit AND n.notification_id > :checkpoint -- where we were interrupted +WHERE n.notification_id BETWEEN :fromid AND :toid +AND n.notification_id <= :cache_limit AND n.notification_id > :checkpoint -- where we were interrupted ORDER BY n.notification_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/embed/state_query.sql b/cmd/ido2icingadb/embed/state_query.sql index 00a92a8d..3e95d482 100644 --- a/cmd/ido2icingadb/embed/state_query.sql +++ b/cmd/ido2icingadb/embed/state_query.sql @@ -3,6 +3,7 @@ SELECT sh.statehistory_id, UNIX_TIMESTAMP(sh.state_time) state_time, sh.state_ti sh.output, sh.long_output, sh.check_source, o.objecttype_id, o.name1, COALESCE(o.name2, '') name2 FROM icinga_statehistory sh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=sh.object_id -WHERE sh.statehistory_id <= :cache_limit AND sh.statehistory_id > :checkpoint -- where we were interrupted +WHERE sh.statehistory_id BETWEEN :fromid AND :toid +AND sh.statehistory_id <= :cache_limit AND sh.statehistory_id > :checkpoint -- where we were interrupted ORDER BY sh.statehistory_id -- this way we know what has already been migrated from just the last row's ID LIMIT :bulk diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 26776844..d0efb0e3 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -23,9 +23,11 @@ import ( "github.com/vbauerster/mpb/v6" "go.uber.org/zap" "golang.org/x/sync/errgroup" + "math" "os" "path" "regexp" + "strings" "time" ) @@ -39,7 +41,11 @@ type Flags struct { // Config defines the YAML config structure. type Config struct { - IDO config.Database `yaml:"ido"` + IDO struct { + config.Database `yaml:"-,inline"` + From int32 `yaml:"from"` + To int32 `yaml:"to" default:"2147483647"` + } `yaml:"ido"` IcingaDB config.Database `yaml:"icingadb"` // Icinga2 specifies information the IDO doesn't provide. Icinga2 struct { @@ -84,6 +90,9 @@ func main() { log.Info("Computing progress") + // Convert Config#IDO.From and .To to IDs to restrict data by PK. + computeIdRange(c) + // computeProgress figures out which data has already been migrated // not to start from the beginning every time in the following migrate(). computeProgress(idb, envId) @@ -162,7 +171,7 @@ func connectAll(c *Config) (ido, idb *icingadb.DB) { eg, _ := errgroup.WithContext(context.Background()) eg.Go(func() error { - ido = connect("IDO", &c.IDO) + ido = connect("IDO", &c.IDO.Database) return nil }) @@ -202,6 +211,45 @@ func startIdoTx(ido *icingadb.DB) { }) } +// computeIdRange initializes types[*].fromId and types[*].toId. +// (On non-recoverable errors the whole program exits.) +func computeIdRange(c *Config) { + types.forEach(func(ht *historyType) { + getBorderId := func(id *uint64, timeColumns []string, compOperator string, borderTime int32, sortOrder string) { + deZeroFied := make([]string, 0, len(timeColumns)) + for _, column := range timeColumns { + deZeroFied = append(deZeroFied, fmt.Sprintf( + "CASE WHEN %[1]s < '1970-01-03 00:00:00' THEN NULL ELSE %[1]s END", column, + )) + } + + var timeExpr string + if len(deZeroFied) > 1 { + timeExpr = "COALESCE(" + strings.Join(deZeroFied, ",") + ")" + } else { + timeExpr = deZeroFied[0] + } + + query := ht.snapshot.Rebind( + "SELECT " + ht.idoIdColumn + " FROM " + ht.idoTable + " WHERE " + timeExpr + " " + compOperator + + " FROM_UNIXTIME(?) ORDER BY " + ht.idoIdColumn + " " + sortOrder + " LIMIT 1", + ) + + switch err := ht.snapshot.Get(id, query, borderTime); err { + case nil, sql.ErrNoRows: + default: + log.With("backend", "IDO", "query", query, "args", []any{borderTime}). + Fatalf("%+v", errors.Wrap(err, "can't perform query")) + } + } + + ht.fromId = math.MaxInt64 + + getBorderId(&ht.fromId, ht.idoEndColumns, ">=", c.IDO.From, "ASC") + getBorderId(&ht.toId, ht.idoStartColumns, "<=", c.IDO.To, "DESC") + }) +} + //go:embed embed/ido_migration_progress_schema.sql var idoMigrationProgressSchema string @@ -238,9 +286,10 @@ func computeProgress(idb *icingadb.DB, envId []byte) { // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. "SELECT CASE WHEN xh."+ht.idoIdColumn+"<=? THEN 1 ELSE 0 END migrated, COUNT(*) cnt FROM "+ - ht.idoTable+" xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id GROUP BY migrated", + ht.idoTable+" xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ + ht.idoIdColumn+" BETWEEN ? AND ? GROUP BY migrated", ), - ht.lastId, + ht.lastId, ht.fromId, ht.toId, ) if err != nil { log.Fatalf("%+v", errors.Wrap(err, "can't count query")) @@ -337,9 +386,9 @@ func migrateOneType[IdoRow any]( // For the case that the cache was older that the IDO, // but ht.cacheFiller couldn't update it, limit (WHERE) our source data set. if ht.cacheLimitQuery != "" { - var limit uint64 + var limit sql.NullInt64 cacheGet(ht.cache, &limit, ht.cacheLimitQuery) - args = map[string]interface{}{"cache_limit": limit} + args = map[string]interface{}{"cache_limit": limit.Int64} } upsertProgress, _ := idb.BuildUpsertStmt(&IdoMigrationProgress{}) @@ -349,7 +398,7 @@ func migrateOneType[IdoRow any]( // Stream IDO rows, ... sliceIdoHistory( - ht.snapshot, ht.migrationQuery, args, ht.lastId, + ht, ht.migrationQuery, args, ht.lastId, func(idoRows []IdoRow) (checkpoint interface{}) { // ... convert them, ... inserts, upserts, lastIdoId := convertRows( diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index a4e59368..e3db0a28 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -173,30 +173,32 @@ func calcServiceId(env, name1, name2 string) []byte { return hashAny([2]string{env, name1 + "!" + name2}) } -// sliceIdoHistory performs query with args+map[string]interface{}{"checkpoint": checkpoint, "bulk": bulk} on snapshot +// sliceIdoHistory performs query with args+fromid,toid,checkpoint,bulk on ht.snapshot // and passes the results to onRows until either an empty result set or onRows() returns nil. // Rationale: split the likely large result set of a query by adding a WHERE condition and a LIMIT, // both with :named placeholders (:checkpoint, :bulk). // checkpoint is the initial value for the WHERE condition, onRows() returns follow-up ones. // (On non-recoverable errors the whole program exits.) func sliceIdoHistory[Row any]( - snapshot *sqlx.Tx, query string, args map[string]interface{}, + ht *historyType, query string, args map[string]any, checkpoint interface{}, onRows func([]Row) (checkpoint interface{}), ) { if args == nil { args = map[string]interface{}{} } + args["fromid"] = ht.fromId + args["toid"] = ht.toId args["checkpoint"] = checkpoint args["bulk"] = 20000 - if snapshot.DriverName() != driver.MySQL { + if ht.snapshot.DriverName() != driver.MySQL { query = strings.ReplaceAll(query, " USE INDEX (PRIMARY)", "") } for { // TODO: use Tx#SelectNamed() one nice day (https://github.com/jmoiron/sqlx/issues/779) - stmt, err := snapshot.PrepareNamed(query) + stmt, err := ht.snapshot.PrepareNamed(query) if err != nil { log.With("query", query).Fatalf("%+v", errors.Wrap(err, "can't prepare query")) } @@ -228,6 +230,10 @@ type historyType struct { idoTable string // idoIdColumn specifies idoTable's primary key. idoIdColumn string + // idoStartColumns specifies idoTable's event start time locations. (First non-NULL is used.) + idoStartColumns []string + // idoEndColumns specifies idoTable's event end time locations. (First non-NULL is used.) + idoEndColumns []string // cacheSchema specifies .sqlite3's structure. cacheSchema string // cacheFiller fills cache from snapshot. @@ -245,6 +251,10 @@ type historyType struct { cache *sqlx.DB // snapshot represents the data source. snapshot *sqlx.Tx + // fromId is the first IDO row ID to migrate. + fromId uint64 + // toId is the last IDO row ID to migrate. + toId uint64 // total summarizes the source data. total int64 // done summarizes the migrated data. @@ -295,28 +305,36 @@ func (hts historyTypes) forEach(f func(*historyType)) { var types = historyTypes{ { - name: "ack & comment", - idoTable: "icinga_commenthistory", - idoIdColumn: "commenthistory_id", + name: "ack & comment", + idoTable: "icinga_commenthistory", + idoIdColumn: "commenthistory_id", + idoStartColumns: []string{"entry_time"}, + // Manual deletion time wins vs. time of expiration which never happens due to manual deletion. + idoEndColumns: []string{"deletion_time", "expiration_time"}, migrationQuery: commentMigrationQuery, migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, ht *historyType) { migrateOneType(c, idb, envId, endpId, ht, convertCommentRows) }, }, { - name: "downtime", - idoTable: "icinga_downtimehistory", - idoIdColumn: "downtimehistory_id", - migrationQuery: downtimeMigrationQuery, + name: "downtime", + idoTable: "icinga_downtimehistory", + idoIdColumn: "downtimehistory_id", + // Fall back to scheduled time if actual time is missing. + idoStartColumns: []string{"actual_start_time", "scheduled_start_time"}, + idoEndColumns: []string{"actual_end_time", "scheduled_end_time"}, + migrationQuery: downtimeMigrationQuery, migrate: func(c *Config, idb *icingadb.DB, envId []byte, endpId [20]byte, ht *historyType) { migrateOneType(c, idb, envId, endpId, ht, convertDowntimeRows) }, }, { - name: "flapping", - idoTable: "icinga_flappinghistory", - idoIdColumn: "flappinghistory_id", - cacheSchema: eventTimeCacheSchema, + name: "flapping", + idoTable: "icinga_flappinghistory", + idoIdColumn: "flappinghistory_id", + idoStartColumns: []string{"event_time"}, + idoEndColumns: []string{"event_time"}, + cacheSchema: eventTimeCacheSchema, cacheFiller: func(ht *historyType) { buildEventTimeCache(ht, []string{ "xh.flappinghistory_id id", "UNIX_TIMESTAMP(xh.event_time) event_time", @@ -329,10 +347,12 @@ var types = historyTypes{ }, }, { - name: "notification", - idoTable: "icinga_notifications", - idoIdColumn: "notification_id", - cacheSchema: previousHardStateCacheSchema, + name: "notification", + idoTable: "icinga_notifications", + idoIdColumn: "notification_id", + idoStartColumns: []string{"start_time"}, + idoEndColumns: []string{"end_time"}, + cacheSchema: previousHardStateCacheSchema, cacheFiller: func(ht *historyType) { buildPreviousHardStateCache(ht, []string{ "xh.notification_id id", "xh.object_id", "xh.state last_hard_state", @@ -345,10 +365,12 @@ var types = historyTypes{ }, }, { - name: "state", - idoTable: "icinga_statehistory", - idoIdColumn: "statehistory_id", - cacheSchema: previousHardStateCacheSchema, + name: "state", + idoTable: "icinga_statehistory", + idoIdColumn: "statehistory_id", + idoStartColumns: []string{"state_time"}, + idoEndColumns: []string{"state_time"}, + cacheSchema: previousHardStateCacheSchema, cacheFiller: func(ht *historyType) { buildPreviousHardStateCache(ht, []string{"xh.statehistory_id id", "xh.object_id", "xh.last_hard_state"}) }, From 7ff52c5c21ce952333fc12768f790d0e79200369 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 7 Sep 2022 14:20:47 +0200 Subject: [PATCH 094/103] cmd/ido2icingadb: handle custom input data time range changes --- .../embed/ido_migration_progress_schema.sql | 4 +- cmd/ido2icingadb/main.go | 43 ++++++++++++------- cmd/ido2icingadb/misc.go | 2 + 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql b/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql index 4049e6ae..54c1c005 100644 --- a/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql +++ b/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql @@ -1,7 +1,9 @@ CREATE TABLE IF NOT EXISTS ido_migration_progress ( environment_id CHAR(40) NOT NULL, -- Hex SHA1. Rationale: CHAR(40) is not RDBMS-specific history_type VARCHAR(63) NOT NULL, + from_ts BIGINT NOT NULL, + to_ts BIGINT NOT NULL, last_ido_id BIGINT NOT NULL, - CONSTRAINT pk_ido_migration_progress PRIMARY KEY (environment_id, history_type) + CONSTRAINT pk_ido_migration_progress PRIMARY KEY (environment_id, history_type, from_ts, to_ts) ); diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index d0efb0e3..83f02571 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -26,6 +26,7 @@ import ( "math" "os" "path" + "path/filepath" "regexp" "strings" "time" @@ -86,7 +87,7 @@ func main() { startIdoTx(ido) // Prepare the directory structure the following fillCache() will need later. - mkCache(f, idb.Mapper) + mkCache(f, c, idb.Mapper) log.Info("Computing progress") @@ -95,7 +96,7 @@ func main() { // computeProgress figures out which data has already been migrated // not to start from the beginning every time in the following migrate(). - computeProgress(idb, envId) + computeProgress(c, idb, envId) // On rationale read buildEventTimeCache() and buildPreviousHardStateCache() docs. log.Info("Filling cache") @@ -105,7 +106,7 @@ func main() { migrate(c, idb, envId) log.Info("Cleaning up cache") - cleanupCache() + cleanupCache(f) } // parseConfig validates the f.Config file and returns the config and -1 or - on failure - nil and an exit code. @@ -135,7 +136,7 @@ var nonWords = regexp.MustCompile(`\W+`) // mkCache ensures /.sqlite3 files are present and contain their schema // and initializes types[*].cache. (On non-recoverable errors the whole program exits.) -func mkCache(f *Flags, mapper *reflectx.Mapper) { +func mkCache(f *Flags, c *Config, mapper *reflectx.Mapper) { log.Info("Preparing cache") if err := os.MkdirAll(f.Cache, 0700); err != nil { @@ -147,7 +148,10 @@ func mkCache(f *Flags, mapper *reflectx.Mapper) { return } - file := path.Join(f.Cache, nonWords.ReplaceAllLiteralString(ht.name, "_")+".sqlite3") + file := path.Join(f.Cache, fmt.Sprintf( + "%s_%d-%d.sqlite3", nonWords.ReplaceAllLiteralString(ht.name, "_"), c.IDO.From, c.IDO.To, + )) + var err error ht.cache, err = sqlx.Open("sqlite3", "file:"+file) @@ -255,7 +259,7 @@ var idoMigrationProgressSchema string // computeProgress initializes types[*].lastId, types[*].total and types[*].done. // (On non-recoverable errors the whole program exits.) -func computeProgress(idb *icingadb.DB, envId []byte) { +func computeProgress(c *Config, idb *icingadb.DB, envId []byte) { if _, err := idb.Exec(idoMigrationProgressSchema); err != nil { log.Fatalf("%+v", errors.Wrap(err, "can't create table ido_migration_progress")) } @@ -263,13 +267,16 @@ func computeProgress(idb *icingadb.DB, envId []byte) { envIdHex := hex.EncodeToString(envId) types.forEach(func(ht *historyType) { var query = idb.Rebind( - "SELECT last_ido_id FROM ido_migration_progress WHERE environment_id=? AND history_type=?", + "SELECT last_ido_id FROM ido_migration_progress" + + " WHERE environment_id=? AND history_type=? AND from_ts=? AND to_ts=?", ) - switch err := idb.Get(&ht.lastId, query, envIdHex, ht.name); err { + args := [...]any{envIdHex, ht.name, c.IDO.From, c.IDO.To} + + switch err := idb.Get(&ht.lastId, query, args[:]...); err { case nil, sql.ErrNoRows: default: - log.With("backend", "Icinga DB", "query", query, "args", []interface{}{ht.name}). + log.With("backend", "Icinga DB", "query", query, "args", args[:]). Fatalf("%+v", errors.Wrap(err, "can't perform query")) } }) @@ -435,7 +442,7 @@ func migrateOneType[IdoRow any]( args := map[string]interface{}{"history_type": ht.name, "last_ido_id": lastIdoId} _, err := idb.NamedExec(upsertProgress, &IdoMigrationProgress{ - IdoMigrationProgressUpserter{lastIdoId}, envIdHex, ht.name, + IdoMigrationProgressUpserter{lastIdoId}, envIdHex, ht.name, c.IDO.From, c.IDO.To, }) if err != nil { log.With("backend", "Icinga DB", "dml", upsertProgress, "args", args). @@ -452,16 +459,22 @@ func migrateOneType[IdoRow any]( } // cleanupCache removes /.sqlite3 files. -func cleanupCache() { +func cleanupCache(f *Flags) { types.forEach(func(ht *historyType) { if ht.cacheFile != "" { if err := ht.cache.Close(); err != nil { log.With("file", ht.cacheFile).Warnf("%+v", errors.Wrap(err, "can't close SQLite database")) } - - if err := os.Remove(ht.cacheFile); err != nil { - log.With("file", ht.cacheFile).Warnf("%+v", errors.Wrap(err, "can't remove SQLite database")) - } } }) + + if matches, err := filepath.Glob(path.Join(f.Cache, "*.sqlite3")); err == nil { + for _, match := range matches { + if err := os.Remove(match); err != nil { + log.With("file", match).Warnf("%+v", errors.Wrap(err, "can't remove SQLite database")) + } + } + } else { + log.With("dir", f.Cache).Warnf("%+v", errors.Wrap(err, "can't list SQLite databases")) + } } diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index e3db0a28..6cedd9ef 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -115,6 +115,8 @@ type IdoMigrationProgress struct { IdoMigrationProgressUpserter `json:",inline"` EnvironmentId string `json:"environment_id"` HistoryType string `json:"history_type"` + FromTs int32 `json:"from_ts"` + ToTs int32 `json:"to_ts"` } // Assert interface compliance. From f7d132ccfa481b2e34e860638fd192142941921e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 11 Oct 2022 10:35:55 +0200 Subject: [PATCH 095/103] Make checkDbSchema() reusable as DB#CheckSchema() --- cmd/icingadb/main.go | 41 ++++------------------------------------- pkg/icingadb/db.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go index d778c412..29b7c444 100644 --- a/cmd/icingadb/main.go +++ b/cmd/icingadb/main.go @@ -6,7 +6,6 @@ import ( "github.com/go-redis/redis/v8" "github.com/icinga/icingadb/internal/command" "github.com/icinga/icingadb/pkg/common" - "github.com/icinga/icingadb/pkg/driver" "github.com/icinga/icingadb/pkg/icingadb" "github.com/icinga/icingadb/pkg/icingadb/history" "github.com/icinga/icingadb/pkg/icingadb/overdue" @@ -29,11 +28,9 @@ import ( ) const ( - ExitSuccess = 0 - ExitFailure = 1 - expectedRedisSchemaVersion = "5" - expectedMysqlSchemaVersion = 3 - expectedPostgresSchemaVersion = 1 + ExitSuccess = 0 + ExitFailure = 1 + expectedRedisSchemaVersion = "5" ) func main() { @@ -74,7 +71,7 @@ func run() int { } } - if err := checkDbSchema(context.Background(), db); err != nil { + if err := db.CheckSchema(context.Background()); err != nil { logger.Fatalf("%+v", err) } @@ -358,36 +355,6 @@ func run() int { } } -// checkDbSchema asserts the database schema of the expected version being present. -func checkDbSchema(ctx context.Context, db *icingadb.DB) error { - var expectedDbSchemaVersion uint16 - switch db.DriverName() { - case driver.MySQL: - expectedDbSchemaVersion = expectedMysqlSchemaVersion - case driver.PostgreSQL: - expectedDbSchemaVersion = expectedPostgresSchemaVersion - } - - var version uint16 - - err := db.QueryRowxContext(ctx, "SELECT version FROM icingadb_schema ORDER BY id DESC LIMIT 1").Scan(&version) - if err != nil { - return errors.Wrap(err, "can't check database schema version") - } - - if version != expectedDbSchemaVersion { - // Since these error messages are trivial and mostly caused by users, we don't need - // to print a stack trace here. However, since errors.Errorf() does this automatically, - // we need to use fmt instead. - return fmt.Errorf( - "unexpected database schema version: v%d (expected v%d), please make sure you have applied all database"+ - " migrations after upgrading Icinga DB", version, expectedDbSchemaVersion, - ) - } - - return nil -} - // monitorRedisSchema monitors rc's icinga:schema version validity. func monitorRedisSchema(logger *logging.Logger, rc *icingaredis.Client, pos string) { for { diff --git a/pkg/icingadb/db.go b/pkg/icingadb/db.go index 4494eaf4..a9eed7fe 100644 --- a/pkg/icingadb/db.go +++ b/pkg/icingadb/db.go @@ -87,6 +87,41 @@ func NewDb(db *sqlx.DB, logger *logging.Logger, options *Options) *DB { } } +const ( + expectedMysqlSchemaVersion = 3 + expectedPostgresSchemaVersion = 1 +) + +// CheckSchema asserts the database schema of the expected version being present. +func (db *DB) CheckSchema(ctx context.Context) error { + var expectedDbSchemaVersion uint16 + switch db.DriverName() { + case driver.MySQL: + expectedDbSchemaVersion = expectedMysqlSchemaVersion + case driver.PostgreSQL: + expectedDbSchemaVersion = expectedPostgresSchemaVersion + } + + var version uint16 + + err := db.QueryRowxContext(ctx, "SELECT version FROM icingadb_schema ORDER BY id DESC LIMIT 1").Scan(&version) + if err != nil { + return errors.Wrap(err, "can't check database schema version") + } + + if version != expectedDbSchemaVersion { + // Since these error messages are trivial and mostly caused by users, we don't need + // to print a stack trace here. However, since errors.Errorf() does this automatically, + // we need to use fmt instead. + return fmt.Errorf( + "unexpected database schema version: v%d (expected v%d), please make sure you have applied all database"+ + " migrations after upgrading Icinga DB", version, expectedDbSchemaVersion, + ) + } + + return nil +} + // BuildColumns returns all columns of the given struct. func (db *DB) BuildColumns(subject interface{}) []string { fields := db.Mapper.TypeMap(reflect.TypeOf(subject)).Names From f233c632098a2801f8d563fadb4124712da5a0ab Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 11 Oct 2022 10:44:21 +0200 Subject: [PATCH 096/103] cmd/ido2icingadb: assert schema --- cmd/ido2icingadb/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 83f02571..e989e39b 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -82,6 +82,10 @@ func main() { ido, idb := connectAll(c) + if err := idb.CheckSchema(context.Background()); err != nil { + log.Fatalf("%+v", err) + } + // Start repeatable-read-isolated transactions (consistent SELECTs) // not to have to care for IDO data changes during migration. startIdoTx(ido) From 5ebb2b0798bfc0e35428287b8332b95cecd27193 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 11 Oct 2022 12:30:22 +0200 Subject: [PATCH 097/103] cmd/ido2icingadb: Keep It Simple Stupid --- cmd/ido2icingadb/main.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index e989e39b..42741ad9 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -275,12 +275,10 @@ func computeProgress(c *Config, idb *icingadb.DB, envId []byte) { " WHERE environment_id=? AND history_type=? AND from_ts=? AND to_ts=?", ) - args := [...]any{envIdHex, ht.name, c.IDO.From, c.IDO.To} + args := []any{envIdHex, ht.name, c.IDO.From, c.IDO.To} - switch err := idb.Get(&ht.lastId, query, args[:]...); err { - case nil, sql.ErrNoRows: - default: - log.With("backend", "Icinga DB", "query", query, "args", args[:]). + if err := idb.Get(&ht.lastId, query, args...); err != nil && err != sql.ErrNoRows { + log.With("backend", "Icinga DB", "query", query, "args", args). Fatalf("%+v", errors.Wrap(err, "can't perform query")) } }) From 016a5ddae13dad965f61e16a9b02ba8ee1bfc939 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 11 Oct 2022 14:49:31 +0200 Subject: [PATCH 098/103] cmd/ido2icingadb: don't tell how to mess up everything --- cmd/ido2icingadb/README.md | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/cmd/ido2icingadb/README.md b/cmd/ido2icingadb/README.md index 395822ec..aec85101 100644 --- a/cmd/ido2icingadb/README.md +++ b/cmd/ido2icingadb/README.md @@ -47,7 +47,7 @@ you will find a copy of this license in the [LICENSE] file. for the period both IDO and Icinga DB are active. Disable the IDO feature -- the sooner, the better! Or read on while not disabling it yet. - There are ways to avoid duplicate events. + There is a way to avoid duplicate events. ### Configuration file @@ -117,31 +117,6 @@ In case of an interrupt re-run. If there is much to migrate, use e.g. tmux to protect yourself against SSH connection losses. -### Avoid duplicate events (optional) - -The easiest option has already been mentioned in the _Input time range_ section. - -Another option is to both enable Icinga DB -and disable IDO within one Icinga 2 reload. - -But if the latter doesn't work on first try, you'll lose history. -Not to lose any data consider everything done above just a test. -Reset Icinga DB and re-migrate as follows: - -!!! warning - - The following assumes you didn't disable the IDO, yet! - -1. Disable the Icinga DB feature: `icinga2 feature disable icingadb` -2. Reload Icinga 2: `systemctl reload icinga2` -3. Stop Icinga DB: `systemctl stop icingadb` -4. Reset Redis: `icingadb-redis-cli flushdb` -5. Re-create Icinga DB's database - (drop database, create database, import schema, grant permissions) -6. Start Icinga DB: `systemctl start icingadb` -7. Both re-enable Icinga DB and disable IDO within one Icinga 2 reload -8. Migrate history again, with a clean cache directory (just to be sure) - ## Support Check the [project website] for status updates. Join the [community channels] From 649f7e0e3dfa9ecf874e5dd4a1447b52bcdc9ead Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 14 Oct 2022 16:58:14 +0200 Subject: [PATCH 099/103] cmd/ido2icingadb: build cache from [0,toId], not just [fromId,toId] Just to be sure. --- cmd/ido2icingadb/cache.go | 4 ++-- cmd/ido2icingadb/main.go | 22 ++++++++++++++++++++-- cmd/ido2icingadb/misc.go | 6 ++++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/cmd/ido2icingadb/cache.go b/cmd/ido2icingadb/cache.go index 8aaaec0e..d1854c9a 100644 --- a/cmd/ido2icingadb/cache.go +++ b/cmd/ido2icingadb/cache.go @@ -49,7 +49,7 @@ func buildEventTimeCache(ht *historyType, idoColumns []string) { // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ - ht.idoIdColumn+" BETWEEN :fromid AND :toid AND xh."+ + ht.idoIdColumn+" <= :toid AND xh."+ ht.idoIdColumn+" > :checkpoint ORDER BY xh."+ht.idoIdColumn+" LIMIT :bulk", nil, checkpoint.MaxId.Int64, // ... since we were interrupted: func(idoRows []row) (checkpoint interface{}) { @@ -160,7 +160,7 @@ func buildPreviousHardStateCache(ht *historyType, idoColumns []string) { // For actual migration icinga_objects will be joined anyway, // so it makes no sense to take vanished objects into account. " xh USE INDEX (PRIMARY) INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ - ht.idoIdColumn+" BETWEEN :fromid AND :toid AND xh."+ + ht.idoIdColumn+" <= :toid AND xh."+ ht.idoIdColumn+" < :checkpoint ORDER BY xh."+ht.idoIdColumn+" DESC LIMIT :bulk", nil, checkpoint, // ... since we were interrupted: func(idoRows []row) (checkpoint interface{}) { diff --git a/cmd/ido2icingadb/main.go b/cmd/ido2icingadb/main.go index 42741ad9..af890fb3 100644 --- a/cmd/ido2icingadb/main.go +++ b/cmd/ido2icingadb/main.go @@ -283,6 +283,24 @@ func computeProgress(c *Config, idb *icingadb.DB, envId []byte) { } }) + types.forEach(func(ht *historyType) { + if ht.cacheFiller != nil { + err := ht.snapshot.Get( + &ht.cacheTotal, + ht.snapshot.Rebind( + // For actual migration icinga_objects will be joined anyway, + // so it makes no sense to take vanished objects into account. + "SELECT COUNT(*) FROM "+ht.idoTable+ + " xh INNER JOIN icinga_objects o ON o.object_id=xh.object_id WHERE xh."+ht.idoIdColumn+" <= ?", + ), + ht.toId, + ) + if err != nil { + log.Fatalf("%+v", errors.Wrap(err, "can't count query")) + } + } + }) + types.forEach(func(ht *historyType) { var rows []struct { Migrated uint8 @@ -321,7 +339,7 @@ func fillCache() { progress := mpb.New() for _, ht := range types { if ht.cacheFiller != nil { - ht.setupBar(progress) + ht.setupBar(progress, ht.cacheTotal) } } @@ -340,7 +358,7 @@ func migrate(c *Config, idb *icingadb.DB, envId []byte) { progress := mpb.New() for _, ht := range types { - ht.setupBar(progress) + ht.setupBar(progress, ht.total) } types.forEach(func(ht *historyType) { diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 6cedd9ef..101b5e9b 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -259,6 +259,8 @@ type historyType struct { toId uint64 // total summarizes the source data. total int64 + // cacheTotal summarizes the cache source data. + cacheTotal int64 // done summarizes the migrated data. done int64 // bar represents the current progress bar. @@ -268,7 +270,7 @@ type historyType struct { } // setupBar (re-)initializes ht.bar. -func (ht *historyType) setupBar(progress *mpb.Progress) { +func (ht *historyType) setupBar(progress *mpb.Progress, total int64) { e := &eta{} ops := &opsPerSec{} @@ -279,7 +281,7 @@ func (ht *historyType) setupBar(progress *mpb.Progress) { ops.Init() ht.bar = progress.AddBar( - ht.total, + total, mpb.BarFillerClearOnComplete(), mpb.PrependDecorators( decor.Name(ht.name, decor.WC{W: len(ht.name) + 1, C: decor.DidentRight}), From 70a0bfb867a12bbc180a956dbb635694ab64b29f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 24 Oct 2022 11:25:57 +0200 Subject: [PATCH 100/103] cmd/ido2icingadb: use built-in decorators for the progress bar --- cmd/ido2icingadb/misc.go | 124 +++++++++------------------------------ 1 file changed, 27 insertions(+), 97 deletions(-) diff --git a/cmd/ido2icingadb/misc.go b/cmd/ido2icingadb/misc.go index 101b5e9b..3163dd8f 100644 --- a/cmd/ido2icingadb/misc.go +++ b/cmd/ido2icingadb/misc.go @@ -3,7 +3,6 @@ package main import ( "context" "crypto/sha1" - "fmt" "github.com/icinga/icingadb/pkg/contracts" "github.com/icinga/icingadb/pkg/driver" "github.com/icinga/icingadb/pkg/icingadb" @@ -19,89 +18,6 @@ import ( "time" ) -// slowlyIncrementingProgressDecorator is the base decorator -// for possibly slowly incrementing progresses possibly starting from >0%. -type slowlyIncrementingProgressDecorator struct { - decor.WC - - // startProgress is the first progress >0 seen by Decor. - startProgress int64 - // startTime tells when is startProgress from. - startTime time.Time - // lastProgress is the last progress >0 seen by Decor. - lastProgress int64 - // lastTime tells when is lastProgress from. - lastTime time.Time -} - -// update is to be called by Decor and returns whether sipd is warmed up. -func (sipd *slowlyIncrementingProgressDecorator) update(s decor.Statistics) bool { - if s.Completed || s.Current < 1 { - return false - } - - if sipd.startProgress < 1 { - sipd.startProgress = s.Current - sipd.startTime = time.Now() - sipd.lastProgress = sipd.startProgress - sipd.lastTime = sipd.startTime - - return false - } - - if s.Current == sipd.startProgress { - return false - } - - if s.Current > sipd.lastProgress { - sipd.lastProgress = s.Current - sipd.lastTime = time.Now() - } - - return true -} - -// eta indicates the ETA for possibly slowly incrementing progresses possibly starting from >0%. -type eta struct { - slowlyIncrementingProgressDecorator -} - -// Decor implements the decor.Decorator interface. -func (e *eta) Decor(s decor.Statistics) string { - if e.update(s) { - timePerItem := float64(e.lastTime.Sub(e.startTime)) / float64(e.lastProgress-e.startProgress) - lastETA := time.Duration(float64(s.Total-s.Current) * timePerItem) - - return e.FormatMsg(((lastETA - time.Since(e.lastTime)) / time.Second * time.Second).String()) - } else { - return "" - } -} - -// eta indicates ops/s for possibly slowly incrementing progresses possibly starting from >0%. -type opsPerSec struct { - slowlyIncrementingProgressDecorator -} - -// Decor implements the decor.Decorator interface. -func (ops *opsPerSec) Decor(s decor.Statistics) string { - if ops.update(s) { - return ops.FormatMsg(fmt.Sprintf( - "%.0f/s", - float64(ops.lastProgress-ops.startProgress)/ - (float64(ops.lastTime.Sub(ops.startTime))/float64(time.Second)), - )) - } else { - return "" - } -} - -// Assert interface compliance. -var ( - _ decor.Decorator = (*eta)(nil) - _ decor.Decorator = (*opsPerSec)(nil) -) - type IdoMigrationProgressUpserter struct { LastIdoId any `json:"last_ido_id"` } @@ -224,6 +140,25 @@ func sliceIdoHistory[Row any]( } } +type progressBar struct { + *mpb.Bar + + lastUpdate time.Time +} + +// IncrBy does pb.Bar.DecoratorEwmaUpdate() automatically. +func (pb *progressBar) IncrBy(n int) { + pb.Bar.IncrBy(n) + + now := time.Now() + + if !pb.lastUpdate.IsZero() { + pb.Bar.DecoratorEwmaUpdate(now.Sub(pb.lastUpdate)) + } + + pb.lastUpdate = now +} + // historyType specifies a history data type. type historyType struct { // name is a human-readable common name. @@ -264,31 +199,26 @@ type historyType struct { // done summarizes the migrated data. done int64 // bar represents the current progress bar. - bar *mpb.Bar + bar *progressBar // lastId is the last already migrated ID. lastId uint64 } // setupBar (re-)initializes ht.bar. func (ht *historyType) setupBar(progress *mpb.Progress, total int64) { - e := &eta{} - ops := &opsPerSec{} - - e.W = 4 - ops.W = 4 - - e.Init() - ops.Init() - - ht.bar = progress.AddBar( + ht.bar = &progressBar{Bar: progress.AddBar( total, mpb.BarFillerClearOnComplete(), mpb.PrependDecorators( decor.Name(ht.name, decor.WC{W: len(ht.name) + 1, C: decor.DidentRight}), decor.Percentage(decor.WC{W: 5}), ), - mpb.AppendDecorators(e, decor.Name(" "), ops), - ) + mpb.AppendDecorators( + decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{W: 4}), + decor.Name(" "), + decor.EwmaSpeed(0, "%.0f/s", 0, decor.WC{W: 4}), + ), + )} } type historyTypes []*historyType From 0ff17284344c1a94f71e832cf2306ba0ca34bd71 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 25 Oct 2022 18:22:59 +0200 Subject: [PATCH 101/103] cmd/ido2icingadb: move docs to the main tree --- .../README.md => doc/06-Migration.md | 61 +++---------------- 1 file changed, 8 insertions(+), 53 deletions(-) rename cmd/ido2icingadb/README.md => doc/06-Migration.md (57%) diff --git a/cmd/ido2icingadb/README.md b/doc/06-Migration.md similarity index 57% rename from cmd/ido2icingadb/README.md rename to doc/06-Migration.md index aec85101..c050fe2a 100644 --- a/cmd/ido2icingadb/README.md +++ b/doc/06-Migration.md @@ -1,20 +1,7 @@ -# Icinga DB Migration +# Migration from IDO -![Icinga Logo] - -#### Table of Contents - -- [About](#about) -- [License](#license) -- [Installation](#installation) -- [Usage](#usage) -- [Support](#support) -- [Contributing](#contributing) - -## About - -This commandline tool migrates history data from [IDO] to [Icinga DB]. -Or, more precisely: from the IDO SQL database to the Icinga DB one. +The Icinga DB Migration commandline tool migrates history data from [IDO] to +Icinga DB. Or, more precisely: from the IDO SQL database to the Icinga DB one. !!! info @@ -22,21 +9,7 @@ Or, more precisely: from the IDO SQL database to the Icinga DB one. Only the past history data of existing IDO setups isn't known to Icinga DB without migration from IDO. -## License - -This tool and its documentation are licensed under the terms -of the GNU General Public License Version 2, -you will find a copy of this license in the [LICENSE] file. - -## Installation - -1. Add the official release repository as described - in the [installation chapter] of the Icinga DB documentation. -2. Install the package `icingadb-migration`. - -## Usage - -### Icinga DB +## Icinga DB 1. Make sure Icinga DB is up, running and writing to its database. 2. Optionally disable Icinga 2's IDO feature. @@ -49,7 +22,7 @@ you will find a copy of this license in the [LICENSE] file. Or read on while not disabling it yet. There is a way to avoid duplicate events. -### Configuration file +## Configuration file Create a YAML file like this somewhere: @@ -80,7 +53,7 @@ icingadb: password: CHANGEME ``` -#### Input time range +### Input time range By default, everything is migrated. If you wish, you can restrict the input data's start and/or end by giving `from` and/or `to` under `ido:` as Unix @@ -95,14 +68,14 @@ Examples: The latter is useful for the range end to avoid duplicate events. -### Cache directory +## Cache directory Choose a (not necessarily yet existing) directory for Icinga DB Migration's internal cache. If either there isn't much to migrate or the migration process won't be interrupted by a reboot (of the machine Icinga DB migration/database runs on), `mktemp -d` is enough. -### Actual migration +## Actual migration Run: @@ -117,23 +90,5 @@ In case of an interrupt re-run. If there is much to migrate, use e.g. tmux to protect yourself against SSH connection losses. -## Support -Check the [project website] for status updates. Join the [community channels] -for questions or ask an Icinga partner for [professional support]. - -## Contributing - -There are many ways to contribute to Icinga -- whether it be sending patches, -testing, reporting bugs, or reviewing and updating the documentation. Every -contribution is appreciated! - - -[Icinga Logo]: https://icinga.com/wp-content/uploads/2014/06/icinga_logo.png [IDO]: https://icinga.com/docs/icinga-2/latest/doc/14-features/#ido-database-db-ido -[Icinga DB]: https://icinga.com/docs/icinga-db/latest/doc/01-About/ -[LICENSE]: ./LICENSE -[installation chapter]: https://icinga.com/docs/icingadb/latest/doc/02-Installation/ -[project website]: https://icinga.com -[community channels]: https://icinga.com/community/ -[professional support]: https://icinga.com/support/ From 6a5f3fd02f92820602b277a3dea23d1a072d60e5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 26 Oct 2022 11:18:52 +0200 Subject: [PATCH 102/103] cmd/: rename ido2icingadb/ to icingadb-migrate/ --- cmd/{ido2icingadb => icingadb-migrate}/cache.go | 0 cmd/{ido2icingadb => icingadb-migrate}/convert.go | 0 cmd/{ido2icingadb => icingadb-migrate}/embed/comment_query.sql | 0 cmd/{ido2icingadb => icingadb-migrate}/embed/downtime_query.sql | 0 .../embed/event_time_cache_schema.sql | 0 cmd/{ido2icingadb => icingadb-migrate}/embed/flapping_query.sql | 0 .../embed/ido_migration_progress_schema.sql | 0 .../embed/notification_query.sql | 0 .../embed/previous_hard_state_cache_schema.sql | 0 cmd/{ido2icingadb => icingadb-migrate}/embed/state_query.sql | 0 cmd/{ido2icingadb => icingadb-migrate}/main.go | 0 cmd/{ido2icingadb => icingadb-migrate}/misc.go | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename cmd/{ido2icingadb => icingadb-migrate}/cache.go (100%) rename cmd/{ido2icingadb => icingadb-migrate}/convert.go (100%) rename cmd/{ido2icingadb => icingadb-migrate}/embed/comment_query.sql (100%) rename cmd/{ido2icingadb => icingadb-migrate}/embed/downtime_query.sql (100%) rename cmd/{ido2icingadb => icingadb-migrate}/embed/event_time_cache_schema.sql (100%) rename cmd/{ido2icingadb => icingadb-migrate}/embed/flapping_query.sql (100%) rename cmd/{ido2icingadb => icingadb-migrate}/embed/ido_migration_progress_schema.sql (100%) rename cmd/{ido2icingadb => icingadb-migrate}/embed/notification_query.sql (100%) rename cmd/{ido2icingadb => icingadb-migrate}/embed/previous_hard_state_cache_schema.sql (100%) rename cmd/{ido2icingadb => icingadb-migrate}/embed/state_query.sql (100%) rename cmd/{ido2icingadb => icingadb-migrate}/main.go (100%) rename cmd/{ido2icingadb => icingadb-migrate}/misc.go (100%) diff --git a/cmd/ido2icingadb/cache.go b/cmd/icingadb-migrate/cache.go similarity index 100% rename from cmd/ido2icingadb/cache.go rename to cmd/icingadb-migrate/cache.go diff --git a/cmd/ido2icingadb/convert.go b/cmd/icingadb-migrate/convert.go similarity index 100% rename from cmd/ido2icingadb/convert.go rename to cmd/icingadb-migrate/convert.go diff --git a/cmd/ido2icingadb/embed/comment_query.sql b/cmd/icingadb-migrate/embed/comment_query.sql similarity index 100% rename from cmd/ido2icingadb/embed/comment_query.sql rename to cmd/icingadb-migrate/embed/comment_query.sql diff --git a/cmd/ido2icingadb/embed/downtime_query.sql b/cmd/icingadb-migrate/embed/downtime_query.sql similarity index 100% rename from cmd/ido2icingadb/embed/downtime_query.sql rename to cmd/icingadb-migrate/embed/downtime_query.sql diff --git a/cmd/ido2icingadb/embed/event_time_cache_schema.sql b/cmd/icingadb-migrate/embed/event_time_cache_schema.sql similarity index 100% rename from cmd/ido2icingadb/embed/event_time_cache_schema.sql rename to cmd/icingadb-migrate/embed/event_time_cache_schema.sql diff --git a/cmd/ido2icingadb/embed/flapping_query.sql b/cmd/icingadb-migrate/embed/flapping_query.sql similarity index 100% rename from cmd/ido2icingadb/embed/flapping_query.sql rename to cmd/icingadb-migrate/embed/flapping_query.sql diff --git a/cmd/ido2icingadb/embed/ido_migration_progress_schema.sql b/cmd/icingadb-migrate/embed/ido_migration_progress_schema.sql similarity index 100% rename from cmd/ido2icingadb/embed/ido_migration_progress_schema.sql rename to cmd/icingadb-migrate/embed/ido_migration_progress_schema.sql diff --git a/cmd/ido2icingadb/embed/notification_query.sql b/cmd/icingadb-migrate/embed/notification_query.sql similarity index 100% rename from cmd/ido2icingadb/embed/notification_query.sql rename to cmd/icingadb-migrate/embed/notification_query.sql diff --git a/cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql b/cmd/icingadb-migrate/embed/previous_hard_state_cache_schema.sql similarity index 100% rename from cmd/ido2icingadb/embed/previous_hard_state_cache_schema.sql rename to cmd/icingadb-migrate/embed/previous_hard_state_cache_schema.sql diff --git a/cmd/ido2icingadb/embed/state_query.sql b/cmd/icingadb-migrate/embed/state_query.sql similarity index 100% rename from cmd/ido2icingadb/embed/state_query.sql rename to cmd/icingadb-migrate/embed/state_query.sql diff --git a/cmd/ido2icingadb/main.go b/cmd/icingadb-migrate/main.go similarity index 100% rename from cmd/ido2icingadb/main.go rename to cmd/icingadb-migrate/main.go diff --git a/cmd/ido2icingadb/misc.go b/cmd/icingadb-migrate/misc.go similarity index 100% rename from cmd/ido2icingadb/misc.go rename to cmd/icingadb-migrate/misc.go From c51d767d7d5d99221d53b6ef4146a97e3b05ff3c Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 26 Oct 2022 15:30:36 +0200 Subject: [PATCH 103/103] Migration: docs enhancements --- doc/06-Migration.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/06-Migration.md b/doc/06-Migration.md index c050fe2a..4a799ba8 100644 --- a/doc/06-Migration.md +++ b/doc/06-Migration.md @@ -18,9 +18,7 @@ Icinga DB. Or, more precisely: from the IDO SQL database to the Icinga DB one. Migration will cause duplicate Icinga DB events for the period both IDO and Icinga DB are active. - Disable the IDO feature -- the sooner, the better! - Or read on while not disabling it yet. - There is a way to avoid duplicate events. + Read on, there is a way to avoid that. ## Configuration file @@ -80,7 +78,7 @@ Icinga DB migration/database runs on), `mktemp -d` is enough. Run: ```shell -icingadb-migrate -c icingadb-migration.yml -t ~/icingadb-migration.tmp +icingadb-migrate -c icingadb-migration.yml -t ~/icingadb-migration.cache ``` In case of an interrupt re-run.