From 09b71d680a511605e51873e66c02f6e83ba08929 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 19 Feb 2019 16:39:37 +0100 Subject: [PATCH 01/69] Add mysql.go & redis.go --- mysql.go | 851 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ redis.go | 274 ++++++++++++++++++ 2 files changed, 1125 insertions(+) create mode 100644 mysql.go create mode 100644 redis.go diff --git a/mysql.go b/mysql.go new file mode 100644 index 00000000..c4adef7c --- /dev/null +++ b/mysql.go @@ -0,0 +1,851 @@ +package connection + +import ( + "container/list" + "context" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "git.icinga.com/icingadb/icingadb/benchmark" + "github.com/go-sql-driver/mysql" + log "github.com/sirupsen/logrus" + oldlog "log" + "io/ioutil" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +type dbTypeBridge interface { + sql.Scanner + Result() interface{} +} + +type dbIntBridge struct { + result interface{} +} + +func (d *dbIntBridge) Scan(src interface{}) (err error) { + baseScanner := sql.NullInt64{} + err = baseScanner.Scan(src) + + if err == nil { + if baseScanner.Valid { + d.result = baseScanner.Int64 + } else { + d.result = nil + } + } + + return +} + +func (d *dbIntBridge) Result() interface{} { + return d.result +} + +type dbFloatBridge struct { + result interface{} +} + +func (d *dbFloatBridge) Scan(src interface{}) (err error) { + baseScanner := sql.NullFloat64{} + err = baseScanner.Scan(src) + + if err == nil { + if baseScanner.Valid { + d.result = baseScanner.Float64 + } else { + d.result = nil + } + } + + return +} + +func (d *dbFloatBridge) Result() interface{} { + return d.result +} + +type dbStringBridge struct { + result interface{} +} + +func (d *dbStringBridge) Scan(src interface{}) (err error) { + baseScanner := sql.NullString{} + err = baseScanner.Scan(src) + + if err == nil { + if baseScanner.Valid { + d.result = baseScanner.String + } else { + d.result = nil + } + } + + return +} + +func (d *dbStringBridge) Result() interface{} { + return d.result +} + +type dbBytesBridge struct { + result interface{} +} + +func (d *dbBytesBridge) Scan(src interface{}) (err error) { + baseScanner := sql.NullString{} + err = baseScanner.Scan(src) + + if err == nil { + if baseScanner.Valid { + d.result = []byte(baseScanner.String) + } else { + d.result = nil + } + } + + return +} + +func (d *dbBytesBridge) Result() interface{} { + return d.result +} + +var dbTypeBridgeFactories = map[string]func() dbTypeBridge{ + // MySQL + "TINYINT": func() dbTypeBridge { + return &dbIntBridge{} + }, + "SMALLINT": func() dbTypeBridge { + return &dbIntBridge{} + }, + "INT": func() dbTypeBridge { + return &dbIntBridge{} + }, + "BIGINT": func() dbTypeBridge { + return &dbIntBridge{} + }, + "FLOAT": func() dbTypeBridge { + return &dbFloatBridge{} + }, + "CHAR": func() dbTypeBridge { + return &dbStringBridge{} + }, + "VARCHAR": func() dbTypeBridge { + return &dbStringBridge{} + }, + "ENUM": func() dbTypeBridge { + return &dbStringBridge{} + }, + "BINARY": func() dbTypeBridge { + return &dbBytesBridge{} + }, + + // SQLite + "INTEGER": func() dbTypeBridge { + return &dbIntBridge{} + }, + "REAL": func() dbTypeBridge { + return &dbFloatBridge{} + }, + "TEXT": func() dbTypeBridge { + return &dbStringBridge{} + }, + "BLOB": func() dbTypeBridge { + return &dbBytesBridge{} + }, + // SELECT 1 FROM ... + "": func() dbTypeBridge { + return &dbIntBridge{} + }, +} + +type dbBrokenBridge struct { + typ string +} + +func (d *dbBrokenBridge) Scan(src interface{}) error { + types := make([]string, len(dbTypeBridgeFactories)) + typeIdx := 0 + + for typ := range dbTypeBridgeFactories { + types[typeIdx] = typ + typeIdx++ + } + + sort.Strings(types) + + return errors.New(fmt.Sprintf("bad column type %s, expected one of %s", d.typ, strings.Join(types, ", "))) +} + +func (d *dbBrokenBridge) Result() interface{} { + return nil +} + +var prettyPrintedSqlReplacer = strings.NewReplacer("\n", " ", "\t", "") + +type prettyPrintedSql struct { + sql string +} + +// String implements and interface from Stringer +func (p prettyPrintedSql) String() string { + return strings.TrimSpace(prettyPrintedSqlReplacer.Replace(p.sql)) +} + +// MarshalText implements an interface from TextMarshaler +func (p prettyPrintedSql) MarshalText() (text []byte, err error) { + return []byte(p.String()), nil +} + +type prettyPrintedArgs struct { + args []interface{} +} + +func (p *prettyPrintedArgs) String() string { + res := "[" + + for _, v := range p.args { + if byteArray, isByteArray := v.([]byte); isByteArray { + res = fmt.Sprintf("%s hex.DecodeString(\"%s\"),", res, hex.EncodeToString(byteArray)) + } else { + res = fmt.Sprintf("%s %#v,", res, v) + } + } + + return res + " ]" +} + +// MarshalText implements an interface from TextMarshaler +func (p prettyPrintedArgs) MarshalText() (text []byte, err error) { + return []byte(p.String()), nil +} + +type prettyPrintedRowsAffected struct { + result sql.Result +} + +// String implements and interface from Stringer +func (d prettyPrintedRowsAffected) String() string { + if d.result != nil { + rows, errRA := d.result.RowsAffected() + if errRA == nil { + return strconv.FormatInt(rows, 10) + } + } + + return "N/A" +} + +// MarshalText implements an interface from TextMarshaler +func (d prettyPrintedRowsAffected) MarshalText() (text []byte, err error) { + return []byte(d.String()), nil +} + +// Either a connection or a transaction +type DbClient interface { + Query(query string, args ...interface{}) (*sql.Rows, error) +} + +// Database wrapper including helper functions +type DBWrapper struct { + Db *sql.DB + ConnectedAtomic *uint32 //uint32 to be able to use atomic operations + ConnectionUpCondition *sync.Cond + ConnectionLostCounter int +} + +func (dbw *DBWrapper) IsConnected() bool { + return *dbw.ConnectedAtomic != 0 +} + +func (dbw *DBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { + if connected { + return atomic.CompareAndSwapUint32(dbw.ConnectedAtomic, 0, 1) + } else { + return atomic.CompareAndSwapUint32(dbw.ConnectedAtomic, 1, 0) + } +} + +// mkMysql creates a new MySQL client. +func mkMysql(dbType string, dbDsn string) (*sql.DB, error) { + log.Info("Connecting to MySQL") + + sep := "?" + + if dbDsn == "" { + dbDsn = "/" + } else { + dsnParts := strings.Split(dbDsn, "/") + if strings.Contains(dsnParts[len(dsnParts)-1], "?") { + sep = "&" + } + } + + dbDsn = dbDsn + sep + + "innodb_strict_mode=1&sql_mode='STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_ENGINE_SUBSTITUTION,PIPES_AS_CONCAT,ANSI_QUOTES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER'" + + db, errConn := sql.Open(dbType, dbDsn) + if errConn != nil { + return nil, errConn + } + + mysql.SetLogger(oldlog.New(ioutil.Discard, "", 0)) + + db.SetMaxOpenConns(100) + db.SetMaxIdleConns(0) + + return db, nil +} + +func NewDBWrapper(dbType string, dbDsn string) (*DBWrapper, error) { + db, err := mkMysql(dbType, dbDsn) + + if err != nil { + return nil, err + } + + dbw := DBWrapper{Db: db, ConnectedAtomic: new(uint32)} + dbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) + + go func() { + for { + dbw.checkConnection(true) + time.Sleep(dbw.getConnectionCheckInterval()) + } + }() + + return &dbw, nil +} + +func (dbw *DBWrapper) getConnectionCheckInterval() time.Duration { + if !dbw.IsConnected() { + if dbw.ConnectionLostCounter < 4 { + return 5 * time.Second + } else if dbw.ConnectionLostCounter < 8 { + return 10 * time.Second + } else if dbw.ConnectionLostCounter < 11 { + return 30 * time.Second + } else if dbw.ConnectionLostCounter < 14 { + return 60 * time.Second + } else { + log.Fatal("Could not connect to SQL for over 5 minutes. Shutting down...") + } + } + + return 15 * time.Second +} + +func (dbw *DBWrapper) Query(query string, args ...interface{}) (*sql.Rows, error) { + return dbw.Db.Query(query, args...) +} + +func (dbw *DBWrapper) checkConnection(isTicker bool) bool { + err := dbw.Db.Ping() + if err != nil { + if dbw.CompareAndSetConnected(false) { + log.WithFields(log.Fields{ + "context": "sql", + "error": err, + }).Error("SQL connection lost. Trying to reconnect") + } else if isTicker { + dbw.ConnectionLostCounter++ + + log.WithFields(log.Fields{ + "context": "sql", + "error": err, + }).Debugf("SQL connection lost. Trying again in %s", dbw.getConnectionCheckInterval()) + } + + return false + } else { + if dbw.CompareAndSetConnected(true) { + log.Info("SQL connection established") + dbw.ConnectionLostCounter = 0 + dbw.ConnectionUpCondition.Broadcast() + } + + return true + } +} + +func (dbw *DBWrapper) WaitForConnection() { + dbw.ConnectionUpCondition.L.Lock() + dbw.ConnectionUpCondition.Wait() + dbw.ConnectionUpCondition.L.Unlock() +} + +type MysqlConnectionError struct { + err string +} + +func (e MysqlConnectionError) Error() string { + return e.err +} + +// SqlTransaction executes the given function inside a transaction. +func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFailure bool, f func(*sql.Tx) error) error { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + benchmarc := benchmark.NewBenchmark() + errTx := dbw.sqlTryTransaction(f, concurrencySafety, false) + benchmarc.Stop() + + //DbIoSeconds.WithLabelValues("mysql", "transaction").Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + }).Debug("Executed transaction") + + if errTx != nil { + //TODO: Do this only for concurrencySafety = true, once we figure out the serialization errors. + if isSerializationFailure(errTx) { + log.WithFields(log.Fields{ + "context": "sql", + "error": errTx, + }).Debug("Repeating transaction") + continue + } + + if !dbw.checkConnection(false) { + if retryOnConnectionFailure { + continue + } else { + return MysqlConnectionError{"Transaction failed duo to a connection error"} + } + } + + log.WithFields(log.Fields{ + "context": "sql", + "error": errTx, + }).Warn("SQL error occurred") + } + + return errTx + } +} + +// SqlTransaction executes the given function inside a transaction. +func (dbw *DBWrapper) SqlTransactionQuiet(concurrencySafety bool, retryOnConnectionFailure bool, f func(*sql.Tx) error) error { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + errTx := dbw.sqlTryTransaction(f, concurrencySafety, true) + if errTx != nil { + //TODO: Do this only for concurrencySafety = true, once we figure out the serialization errors. + if isSerializationFailure(errTx) { + continue + } + + if !dbw.checkConnection(false) { + if retryOnConnectionFailure { + continue + } else { + return MysqlConnectionError{"Transaction failed duo to a connection error"} + } + } + + // We still log errors + log.WithFields(log.Fields{ + "context": "sql", + "error": errTx, + }).Warn("SQL error occurred") + } + return errTx + } +} + +// Executes the given function inside a transaction +func (dbw *DBWrapper) sqlTryTransaction(f func(*sql.Tx) error, concurrencySafety bool, quiet bool) error { + tx, errBegin := dbw.sqlBegin(concurrencySafety, quiet) + if errBegin != nil { + return errBegin + } + + errTx := f(tx) + if errTx != nil { + dbw.sqlRollback(tx, quiet) + return errTx + } + + return dbw.sqlCommit(tx, quiet) +} + +// Returns whether the given error signals serialization failure +// https://dev.mysql.com/doc/refman/5.5/en/error-messages-server.html#error_er_lock_deadlock +func isSerializationFailure(e error) bool { + switch err := e.(type) { + case *mysql.MySQLError: + switch err.Number { + // Those are the error numbers for serialization failures, upon which we retry + case 1205, 1213: + return true + } + } + + return false +} + +// Wrapper around Db.BeginTx() for auto-logging +func (dbw *DBWrapper) sqlBegin(concurrencySafety bool, quiet bool) (*sql.Tx, error) { + var isoLvl sql.IsolationLevel + if concurrencySafety { + isoLvl = sql.LevelSerializable + } else { + isoLvl = sql.LevelReadCommitted + } + + var err error + var tx *sql.Tx + if quiet { + tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) + } else { + benchmarc := benchmark.NewBenchmark() + tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) + benchmarc.Stop() + + //DbIoSeconds.WithLabelValues("mysql", "begin").Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + }).Debug("BEGIN transaction") + } + + return tx, err +} + +// Wrapper around tx.Commit() for auto-logging +func (dbw *DBWrapper) sqlCommit(tx *sql.Tx, quiet bool) error { + var err error + if quiet { + err = tx.Commit() + } else { + benchmarc := benchmark.NewBenchmark() + err = tx.Commit() + benchmarc.Stop() + + //DbIoSeconds.WithLabelValues("mysql", "commit").Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + }).Debug("COMMIT transaction") + } + + return err +} + +// Wrapper around tx.Rollback() for auto-logging +func (dbw *DBWrapper) sqlRollback(tx *sql.Tx, quiet bool) error { + var err error + if !quiet { + benchmarc := benchmark.NewBenchmark() + err = tx.Rollback() + benchmarc.Stop() + + //DbIoSeconds.WithLabelValues("mysql", "rollback").Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + }).Debug("ROLLBACK transaction") + } else { + err = tx.Rollback() + } + + return err +} + +// Wrapper around Db.SqlQuery() for auto-logging +func (dbw *DBWrapper) SqlFetchAll(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + res, err := sqlTryFetchAll(db, queryDescription, query, args...) + + if err != nil { + if _, isDb := db.(*sql.DB); isDb { + if !dbw.checkConnection(false) { + continue + } + } + } + + return res, err + } +} + +func sqlTryFetchAll(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + benchmarc := benchmark.NewBenchmark() + rows, errQuery := db.Query(query, args...) + benchmarc.Stop() + + //DbIoSeconds.WithLabelValues("mysql", queryDescription).Observe(benchmarc.Seconds()) + + rowsCount := 0 + + defer func() { + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + "query": prettyPrintedSql{query}, + "args": prettyPrintedArgs{args}, + "affected_Rows": rowsCount, + }).Debug("Finished FetchAll") + }() + + if errQuery != nil { + return [][]interface{}{}, errQuery + } + + defer rows.Close() + + columnTypes, errCT := rows.ColumnTypes() + if errCT != nil { + return [][]interface{}{}, errCT + } + + colsPerRow := len(columnTypes) + buf := list.New() + bridges := make([]dbTypeBridge, colsPerRow) + scanDest := make([]interface{}, colsPerRow) + + for i, columnType := range columnTypes { + typ := columnType.DatabaseTypeName() + factory, hasFactory := dbTypeBridgeFactories[typ] + if hasFactory { + bridges[i] = factory() + } else { + bridges[i] = &dbBrokenBridge{typ: typ} + } + + scanDest[i] = bridges[i] + } + + for { + if rows.Next() { + if errScan := rows.Scan(scanDest...); errScan != nil { + return [][]interface{}{}, errScan + } + + row := make([]interface{}, colsPerRow) + + for i, bridge := range bridges { + row[i] = bridge.Result() + } + + buf.PushBack(row) + } else if errNx := rows.Err(); errNx == nil { + break + } else { + return nil, errNx + } + } + + res := make([][]interface{}, buf.Len()) + + for current, i := buf.Front(), 0; current != nil; current = current.Next() { + res[i] = current.Value.([]interface{}) + i++ + } + + rowsCount = len(res) + + return res, nil +} + +// No logging, no benchmarking +func (dbw *DBWrapper) SqlFetchAllQuiet(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + res, err := sqlTryFetchAllQuiet(db, queryDescription, query, args...) + + if err != nil { + if _, isDb := db.(*sql.DB); isDb { + if !dbw.checkConnection(false) { + continue + } + } + } + + return res, err + } +} + +func sqlTryFetchAllQuiet(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + rows, errQuery := db.Query(query, args...) + + if errQuery != nil { + return [][]interface{}{}, errQuery + } + + defer rows.Close() + + columnTypes, errCT := rows.ColumnTypes() + if errCT != nil { + return [][]interface{}{}, errCT + } + + colsPerRow := len(columnTypes) + buf := list.New() + bridges := make([]dbTypeBridge, colsPerRow) + scanDest := make([]interface{}, colsPerRow) + + for i, columnType := range columnTypes { + typ := columnType.DatabaseTypeName() + factory, hasFactory := dbTypeBridgeFactories[typ] + if hasFactory { + bridges[i] = factory() + } else { + bridges[i] = &dbBrokenBridge{typ: typ} + } + + scanDest[i] = bridges[i] + } + + for { + if rows.Next() { + if errScan := rows.Scan(scanDest...); errScan != nil { + return [][]interface{}{}, errScan + } + + row := make([]interface{}, colsPerRow) + + for i, bridge := range bridges { + row[i] = bridge.Result() + } + + buf.PushBack(row) + } else if errNx := rows.Err(); errNx == nil { + break + } else { + return nil, errNx + } + } + + res := make([][]interface{}, buf.Len()) + + for current, i := buf.Front(), 0; current != nil; current = current.Next() { + res[i] = current.Value.([]interface{}) + i++ + } + + return res, nil +} + +// Wrapper around tx.SqlExec() for auto-logging +func (dbw *DBWrapper) SqlExec(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { + benchmarc := benchmark.NewBenchmark() + res, err := tx.Exec(sql, args...) + benchmarc.Stop() + + //DbIoSeconds.WithLabelValues("mysql", opDescription).Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + "affected_rows": prettyPrintedRowsAffected{res}, + "args": prettyPrintedArgs{args}, + "query": prettyPrintedSql{sql}, + }).Debug("Finished Exec") + + return res, err +} + +// No logging, no benchmarking +func (dbw *DBWrapper) SqlExecQuiet(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { + res, err := tx.Exec(sql, args...) + return res, err +} + +func formatLogQuery(query string) string { + r := strings.NewReplacer("\n", " ", "\t", "") + return strings.TrimSpace(r.Replace(query)) +} + +// Go bool -> DB bool +var yesNo = map[bool]string{ + true: "y", + false: "n", +} + +func ConvertValueForDb(in interface{}) interface{} { + switch value := in.(type) { + case []byte: + case string: + case float64: + case int64: + case nil: + break + case float32: + return float64(value) + case uint: + return int64(value) + case uint8: + return int64(value) + case uint16: + return int64(value) + case uint32: + return int64(value) + case uint64: + return int64(value) + case int: + return int64(value) + case int8: + return int64(value) + case int16: + return int64(value) + case int32: + return int64(value) + case bool: + return yesNo[value] + default: + panic(fmt.Sprintf( + "bad type %s, expected one of []byte, string, float{32,64}, {,u}int{,8,16,32,64}, bool, nil", + reflect.TypeOf(in).Name(), + )) + } + + return in +} + +func MakePlaceholderList(x int) string { + runes := make([]rune, 1+x*2) + + i := 1 + for j := 0; j < x; j++ { + runes[i] = '?' + i++ + + runes[i] = ',' + i++ + } + + runes[0] = '(' + runes[len(runes)-1] = ')' + + return string(runes) +} diff --git a/redis.go b/redis.go new file mode 100644 index 00000000..04323c93 --- /dev/null +++ b/redis.go @@ -0,0 +1,274 @@ +package connection + +import ( + "git.icinga.com/icingadb/icingadb/benchmark" + "github.com/go-redis/redis" + log "github.com/sirupsen/logrus" + "sync" + "sync/atomic" + "time" +) + +type Environment struct { + ID []byte + Name string +} + +type Icinga2RedisWriterEventsConfig struct { + Update, Delete, Dump string +} + +type Icinga2RedisWriterKeyPrefixesConfig struct { + Checksum, Object, Customvar string +} + +type Icinga2RedisWriterKeyPrefixesStatus struct { + Object string +} + +type Icinga2RedisWriterEvents struct { + Config Icinga2RedisWriterEventsConfig + Stats string +} + +type Icinga2RedisWriterKeyPrefixes struct { + Config Icinga2RedisWriterKeyPrefixesConfig + Status Icinga2RedisWriterKeyPrefixesStatus +} + +type Icinga2RedisWriter struct { + Events Icinga2RedisWriterEvents + KeyPrefixes Icinga2RedisWriterKeyPrefixes +} + +var RedisWriter = Icinga2RedisWriter{ + Events: Icinga2RedisWriterEvents{ + Config: Icinga2RedisWriterEventsConfig{ + Dump: "icinga:config:dump", + Delete: "icinga:config:delete", + Update: "icinga:config:update", + }, + Stats: "icinga:stats", + }, + KeyPrefixes: Icinga2RedisWriterKeyPrefixes{ + Config: Icinga2RedisWriterKeyPrefixesConfig{ + Checksum: "icinga:config:checksum:", + Object: "icinga:config:object:", + Customvar: "icinga:config:customvar:", + }, + Status: Icinga2RedisWriterKeyPrefixesStatus{ + Object: "icinga:state:object:", + }, + }, +} + +// Redis wrapper including helper functions +type RDBWrapper struct { + Rdb *redis.Client + ConnectedAtomic *uint32 //uint32 to be able to use atomic operations + ConnectionUpCondition *sync.Cond + ConnectionLostCounter int +} + +func (rdbw *RDBWrapper) IsConnected() bool { + return *rdbw.ConnectedAtomic != 0 +} + +func (rdbw *RDBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { + if connected { + return atomic.CompareAndSwapUint32(rdbw.ConnectedAtomic, 0, 1) + } else { + return atomic.CompareAndSwapUint32(rdbw.ConnectedAtomic, 1, 0) + } +} + +func NewRDBWrapper(rdb *redis.Client) *RDBWrapper { + rdbw := RDBWrapper{Rdb: rdb, ConnectedAtomic: new(uint32)} + rdbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) + + go func() { + for { + rdbw.CheckConnection(true) + time.Sleep(rdbw.getConnectionCheckInterval()) + } + }() + + return &rdbw +} + +func (rdbw *RDBWrapper) getConnectionCheckInterval() time.Duration { + if !rdbw.IsConnected() { + if rdbw.ConnectionLostCounter < 4 { + return 5 * time.Second + } else if rdbw.ConnectionLostCounter < 8 { + return 10 * time.Second + } else if rdbw.ConnectionLostCounter < 11 { + return 30 * time.Second + } else if rdbw.ConnectionLostCounter < 14 { + return 60 * time.Second + } else { + log.Fatal("Could not connect to Redis for over 5 minutes. Shutting down...") + } + } + + return 15 * time.Second +} + +func (rdbw *RDBWrapper) CheckConnection(isTicker bool) bool { + _, err := rdbw.Rdb.Ping().Result() + if err != nil { + if rdbw.CompareAndSetConnected(false) { + log.WithFields(log.Fields{ + "context": "redis", + "error": err, + }).Error("Redis connection lost. Trying to reconnect") + } else if isTicker { + rdbw.ConnectionLostCounter++ + + log.WithFields(log.Fields{ + "context": "redis", + "error": err, + }).Debugf("Redis connection lost. Trying again in %s", rdbw.getConnectionCheckInterval()) + } + + return false + } else { + if rdbw.CompareAndSetConnected(true) { + log.Info("Redis connection established") + rdbw.ConnectionLostCounter = 0 + rdbw.ConnectionUpCondition.Broadcast() + } + + return true + } +} + +func (rdbw *RDBWrapper) WaitForConnection() { + rdbw.ConnectionUpCondition.L.Lock() + rdbw.ConnectionUpCondition.Wait() + rdbw.ConnectionUpCondition.L.Unlock() +} + +// Wrapper for connection handling +func (rdbw *RDBWrapper) Publish(channel string, message interface{}) *redis.IntCmd { + for { + if !rdbw.IsConnected() { + rdbw.WaitForConnection() + continue + } + + cmd := rdbw.Rdb.Publish(channel, message) + _, err := cmd.Result() + + if err != nil { + if !rdbw.CheckConnection(false) { + continue + } + } + + return cmd + } +} + +// Wrapper for connection handling +func (rdbw *RDBWrapper) XRead(args *redis.XReadArgs) *redis.XStreamSliceCmd { + for { + if !rdbw.IsConnected() { + rdbw.WaitForConnection() + continue + } + + cmd := rdbw.Rdb.XRead(args) + _, err := cmd.Result() + + if err != nil { + if !rdbw.CheckConnection(false) { + continue + } + } + + return cmd + } +} +// Wrapper for connection handling +func (rdbw *RDBWrapper) XDel(stream string, ids ...string) *redis.IntCmd { + for { + if !rdbw.IsConnected() { + rdbw.WaitForConnection() + continue + } + + cmd := rdbw.Rdb.XDel(stream, ids...) + _, err := cmd.Result() + + if err != nil { + if !rdbw.CheckConnection(false) { + continue + } + } + + return cmd + } +} + +// Wrapper for auto-logging and connection handling +func (rdbw *RDBWrapper) HGetAll(key string) (map[string]string, error) { + for { + if !rdbw.IsConnected() { + rdbw.WaitForConnection() + continue + } + + benchmarc := benchmark.NewBenchmark() + res, errHGA := rdbw.Rdb.HGetAll(key).Result() + + if errHGA != nil { + if !rdbw.CheckConnection(false) { + continue + } + } + + benchmarc.Stop() + + //DbIoSeconds.WithLabelValues("redis", "hgetall").Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "redis", + "benchmark": benchmarc, + "query": "HGETALL " + key, + "result": res, + }).Debug("Ran Query") + + return res, errHGA + } +} + +// Wrapper for auto-logging and connection handling +func (rdbw *RDBWrapper) TxPipelined(fn func(pipeliner redis.Pipeliner) error) ([]redis.Cmder, error) { + for { + if !rdbw.IsConnected() { + rdbw.WaitForConnection() + continue + } + benchmarc := benchmark.NewBenchmark() + c, e := rdbw.Rdb.TxPipelined(fn) + + if e != nil { + if !rdbw.CheckConnection(false) { + continue + } + } + + benchmarc.Stop() + + //DbIoSeconds.WithLabelValues("redis", "multi").Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "redis", + "benchmark": benchmarc, + "query": "MULTI/EXEC", + }).Debug("Ran pipelined transaction") + + return c, e + } +} From 829896a1f42067e5825600b764aba585d633fd8d Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 20 Feb 2019 14:43:00 +0100 Subject: [PATCH 02/69] Change package to icingadb_connection --- mysql.go | 2 +- redis.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql.go b/mysql.go index c4adef7c..9e6594ed 100644 --- a/mysql.go +++ b/mysql.go @@ -1,4 +1,4 @@ -package connection +package icingadb_connection import ( "container/list" diff --git a/redis.go b/redis.go index 04323c93..3aa42c5b 100644 --- a/redis.go +++ b/redis.go @@ -1,4 +1,4 @@ -package connection +package icingadb_connection import ( "git.icinga.com/icingadb/icingadb/benchmark" From 994194d095f77b0b9beacb2f3e158a8642937a74 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 20 Feb 2019 14:43:22 +0100 Subject: [PATCH 03/69] Move mysql util functions into mysql_utils.go --- mysql.go | 508 ++----------------------------------------------- mysql_utils.go | 367 +++++++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+), 488 deletions(-) create mode 100644 mysql_utils.go diff --git a/mysql.go b/mysql.go index 9e6594ed..c866a33a 100644 --- a/mysql.go +++ b/mysql.go @@ -4,251 +4,13 @@ import ( "container/list" "context" "database/sql" - "encoding/hex" - "errors" - "fmt" "git.icinga.com/icingadb/icingadb/benchmark" - "github.com/go-sql-driver/mysql" log "github.com/sirupsen/logrus" - oldlog "log" - "io/ioutil" - "reflect" - "sort" - "strconv" - "strings" "sync" "sync/atomic" "time" ) -type dbTypeBridge interface { - sql.Scanner - Result() interface{} -} - -type dbIntBridge struct { - result interface{} -} - -func (d *dbIntBridge) Scan(src interface{}) (err error) { - baseScanner := sql.NullInt64{} - err = baseScanner.Scan(src) - - if err == nil { - if baseScanner.Valid { - d.result = baseScanner.Int64 - } else { - d.result = nil - } - } - - return -} - -func (d *dbIntBridge) Result() interface{} { - return d.result -} - -type dbFloatBridge struct { - result interface{} -} - -func (d *dbFloatBridge) Scan(src interface{}) (err error) { - baseScanner := sql.NullFloat64{} - err = baseScanner.Scan(src) - - if err == nil { - if baseScanner.Valid { - d.result = baseScanner.Float64 - } else { - d.result = nil - } - } - - return -} - -func (d *dbFloatBridge) Result() interface{} { - return d.result -} - -type dbStringBridge struct { - result interface{} -} - -func (d *dbStringBridge) Scan(src interface{}) (err error) { - baseScanner := sql.NullString{} - err = baseScanner.Scan(src) - - if err == nil { - if baseScanner.Valid { - d.result = baseScanner.String - } else { - d.result = nil - } - } - - return -} - -func (d *dbStringBridge) Result() interface{} { - return d.result -} - -type dbBytesBridge struct { - result interface{} -} - -func (d *dbBytesBridge) Scan(src interface{}) (err error) { - baseScanner := sql.NullString{} - err = baseScanner.Scan(src) - - if err == nil { - if baseScanner.Valid { - d.result = []byte(baseScanner.String) - } else { - d.result = nil - } - } - - return -} - -func (d *dbBytesBridge) Result() interface{} { - return d.result -} - -var dbTypeBridgeFactories = map[string]func() dbTypeBridge{ - // MySQL - "TINYINT": func() dbTypeBridge { - return &dbIntBridge{} - }, - "SMALLINT": func() dbTypeBridge { - return &dbIntBridge{} - }, - "INT": func() dbTypeBridge { - return &dbIntBridge{} - }, - "BIGINT": func() dbTypeBridge { - return &dbIntBridge{} - }, - "FLOAT": func() dbTypeBridge { - return &dbFloatBridge{} - }, - "CHAR": func() dbTypeBridge { - return &dbStringBridge{} - }, - "VARCHAR": func() dbTypeBridge { - return &dbStringBridge{} - }, - "ENUM": func() dbTypeBridge { - return &dbStringBridge{} - }, - "BINARY": func() dbTypeBridge { - return &dbBytesBridge{} - }, - - // SQLite - "INTEGER": func() dbTypeBridge { - return &dbIntBridge{} - }, - "REAL": func() dbTypeBridge { - return &dbFloatBridge{} - }, - "TEXT": func() dbTypeBridge { - return &dbStringBridge{} - }, - "BLOB": func() dbTypeBridge { - return &dbBytesBridge{} - }, - // SELECT 1 FROM ... - "": func() dbTypeBridge { - return &dbIntBridge{} - }, -} - -type dbBrokenBridge struct { - typ string -} - -func (d *dbBrokenBridge) Scan(src interface{}) error { - types := make([]string, len(dbTypeBridgeFactories)) - typeIdx := 0 - - for typ := range dbTypeBridgeFactories { - types[typeIdx] = typ - typeIdx++ - } - - sort.Strings(types) - - return errors.New(fmt.Sprintf("bad column type %s, expected one of %s", d.typ, strings.Join(types, ", "))) -} - -func (d *dbBrokenBridge) Result() interface{} { - return nil -} - -var prettyPrintedSqlReplacer = strings.NewReplacer("\n", " ", "\t", "") - -type prettyPrintedSql struct { - sql string -} - -// String implements and interface from Stringer -func (p prettyPrintedSql) String() string { - return strings.TrimSpace(prettyPrintedSqlReplacer.Replace(p.sql)) -} - -// MarshalText implements an interface from TextMarshaler -func (p prettyPrintedSql) MarshalText() (text []byte, err error) { - return []byte(p.String()), nil -} - -type prettyPrintedArgs struct { - args []interface{} -} - -func (p *prettyPrintedArgs) String() string { - res := "[" - - for _, v := range p.args { - if byteArray, isByteArray := v.([]byte); isByteArray { - res = fmt.Sprintf("%s hex.DecodeString(\"%s\"),", res, hex.EncodeToString(byteArray)) - } else { - res = fmt.Sprintf("%s %#v,", res, v) - } - } - - return res + " ]" -} - -// MarshalText implements an interface from TextMarshaler -func (p prettyPrintedArgs) MarshalText() (text []byte, err error) { - return []byte(p.String()), nil -} - -type prettyPrintedRowsAffected struct { - result sql.Result -} - -// String implements and interface from Stringer -func (d prettyPrintedRowsAffected) String() string { - if d.result != nil { - rows, errRA := d.result.RowsAffected() - if errRA == nil { - return strconv.FormatInt(rows, 10) - } - } - - return "N/A" -} - -// MarshalText implements an interface from TextMarshaler -func (d prettyPrintedRowsAffected) MarshalText() (text []byte, err error) { - return []byte(d.String()), nil -} - // Either a connection or a transaction type DbClient interface { Query(query string, args ...interface{}) (*sql.Rows, error) @@ -274,37 +36,6 @@ func (dbw *DBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { } } -// mkMysql creates a new MySQL client. -func mkMysql(dbType string, dbDsn string) (*sql.DB, error) { - log.Info("Connecting to MySQL") - - sep := "?" - - if dbDsn == "" { - dbDsn = "/" - } else { - dsnParts := strings.Split(dbDsn, "/") - if strings.Contains(dsnParts[len(dsnParts)-1], "?") { - sep = "&" - } - } - - dbDsn = dbDsn + sep + - "innodb_strict_mode=1&sql_mode='STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_ENGINE_SUBSTITUTION,PIPES_AS_CONCAT,ANSI_QUOTES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER'" - - db, errConn := sql.Open(dbType, dbDsn) - if errConn != nil { - return nil, errConn - } - - mysql.SetLogger(oldlog.New(ioutil.Discard, "", 0)) - - db.SetMaxOpenConns(100) - db.SetMaxIdleConns(0) - - return db, nil -} - func NewDBWrapper(dbType string, dbDsn string) (*DBWrapper, error) { db, err := mkMysql(dbType, dbDsn) @@ -344,7 +75,22 @@ func (dbw *DBWrapper) getConnectionCheckInterval() time.Duration { } func (dbw *DBWrapper) Query(query string, args ...interface{}) (*sql.Rows, error) { - return dbw.Db.Query(query, args...) + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + res, err := dbw.Db.Query(query, args...) + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + return res, err + } } func (dbw *DBWrapper) checkConnection(isTicker bool) bool { @@ -382,14 +128,6 @@ func (dbw *DBWrapper) WaitForConnection() { dbw.ConnectionUpCondition.L.Unlock() } -type MysqlConnectionError struct { - err string -} - -func (e MysqlConnectionError) Error() string { - return e.err -} - // SqlTransaction executes the given function inside a transaction. func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFailure bool, f func(*sql.Tx) error) error { for { @@ -437,42 +175,9 @@ func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFai } } -// SqlTransaction executes the given function inside a transaction. -func (dbw *DBWrapper) SqlTransactionQuiet(concurrencySafety bool, retryOnConnectionFailure bool, f func(*sql.Tx) error) error { - for { - if !dbw.IsConnected() { - dbw.WaitForConnection() - continue - } - - errTx := dbw.sqlTryTransaction(f, concurrencySafety, true) - if errTx != nil { - //TODO: Do this only for concurrencySafety = true, once we figure out the serialization errors. - if isSerializationFailure(errTx) { - continue - } - - if !dbw.checkConnection(false) { - if retryOnConnectionFailure { - continue - } else { - return MysqlConnectionError{"Transaction failed duo to a connection error"} - } - } - - // We still log errors - log.WithFields(log.Fields{ - "context": "sql", - "error": errTx, - }).Warn("SQL error occurred") - } - return errTx - } -} - // Executes the given function inside a transaction func (dbw *DBWrapper) sqlTryTransaction(f func(*sql.Tx) error, concurrencySafety bool, quiet bool) error { - tx, errBegin := dbw.sqlBegin(concurrencySafety, quiet) + tx, errBegin := dbw.SqlBegin(concurrencySafety, quiet) if errBegin != nil { return errBegin } @@ -483,26 +188,11 @@ func (dbw *DBWrapper) sqlTryTransaction(f func(*sql.Tx) error, concurrencySafety return errTx } - return dbw.sqlCommit(tx, quiet) -} - -// Returns whether the given error signals serialization failure -// https://dev.mysql.com/doc/refman/5.5/en/error-messages-server.html#error_er_lock_deadlock -func isSerializationFailure(e error) bool { - switch err := e.(type) { - case *mysql.MySQLError: - switch err.Number { - // Those are the error numbers for serialization failures, upon which we retry - case 1205, 1213: - return true - } - } - - return false + return dbw.SqlCommit(tx, quiet) } // Wrapper around Db.BeginTx() for auto-logging -func (dbw *DBWrapper) sqlBegin(concurrencySafety bool, quiet bool) (*sql.Tx, error) { +func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (*sql.Tx, error) { var isoLvl sql.IsolationLevel if concurrencySafety { isoLvl = sql.LevelSerializable @@ -531,7 +221,7 @@ func (dbw *DBWrapper) sqlBegin(concurrencySafety bool, quiet bool) (*sql.Tx, err } // Wrapper around tx.Commit() for auto-logging -func (dbw *DBWrapper) sqlCommit(tx *sql.Tx, quiet bool) error { +func (dbw *DBWrapper) SqlCommit(tx *sql.Tx, quiet bool) error { var err error if quiet { err = tx.Commit() @@ -673,89 +363,6 @@ func sqlTryFetchAll(db DbClient, queryDescription string, query string, args ... return res, nil } -// No logging, no benchmarking -func (dbw *DBWrapper) SqlFetchAllQuiet(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { - for { - if !dbw.IsConnected() { - dbw.WaitForConnection() - continue - } - - res, err := sqlTryFetchAllQuiet(db, queryDescription, query, args...) - - if err != nil { - if _, isDb := db.(*sql.DB); isDb { - if !dbw.checkConnection(false) { - continue - } - } - } - - return res, err - } -} - -func sqlTryFetchAllQuiet(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { - rows, errQuery := db.Query(query, args...) - - if errQuery != nil { - return [][]interface{}{}, errQuery - } - - defer rows.Close() - - columnTypes, errCT := rows.ColumnTypes() - if errCT != nil { - return [][]interface{}{}, errCT - } - - colsPerRow := len(columnTypes) - buf := list.New() - bridges := make([]dbTypeBridge, colsPerRow) - scanDest := make([]interface{}, colsPerRow) - - for i, columnType := range columnTypes { - typ := columnType.DatabaseTypeName() - factory, hasFactory := dbTypeBridgeFactories[typ] - if hasFactory { - bridges[i] = factory() - } else { - bridges[i] = &dbBrokenBridge{typ: typ} - } - - scanDest[i] = bridges[i] - } - - for { - if rows.Next() { - if errScan := rows.Scan(scanDest...); errScan != nil { - return [][]interface{}{}, errScan - } - - row := make([]interface{}, colsPerRow) - - for i, bridge := range bridges { - row[i] = bridge.Result() - } - - buf.PushBack(row) - } else if errNx := rows.Err(); errNx == nil { - break - } else { - return nil, errNx - } - } - - res := make([][]interface{}, buf.Len()) - - for current, i := buf.Front(), 0; current != nil; current = current.Next() { - res[i] = current.Value.([]interface{}) - i++ - } - - return res, nil -} - // Wrapper around tx.SqlExec() for auto-logging func (dbw *DBWrapper) SqlExec(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { benchmarc := benchmark.NewBenchmark() @@ -774,78 +381,3 @@ func (dbw *DBWrapper) SqlExec(tx *sql.Tx, opDescription string, sql string, args return res, err } - -// No logging, no benchmarking -func (dbw *DBWrapper) SqlExecQuiet(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { - res, err := tx.Exec(sql, args...) - return res, err -} - -func formatLogQuery(query string) string { - r := strings.NewReplacer("\n", " ", "\t", "") - return strings.TrimSpace(r.Replace(query)) -} - -// Go bool -> DB bool -var yesNo = map[bool]string{ - true: "y", - false: "n", -} - -func ConvertValueForDb(in interface{}) interface{} { - switch value := in.(type) { - case []byte: - case string: - case float64: - case int64: - case nil: - break - case float32: - return float64(value) - case uint: - return int64(value) - case uint8: - return int64(value) - case uint16: - return int64(value) - case uint32: - return int64(value) - case uint64: - return int64(value) - case int: - return int64(value) - case int8: - return int64(value) - case int16: - return int64(value) - case int32: - return int64(value) - case bool: - return yesNo[value] - default: - panic(fmt.Sprintf( - "bad type %s, expected one of []byte, string, float{32,64}, {,u}int{,8,16,32,64}, bool, nil", - reflect.TypeOf(in).Name(), - )) - } - - return in -} - -func MakePlaceholderList(x int) string { - runes := make([]rune, 1+x*2) - - i := 1 - for j := 0; j < x; j++ { - runes[i] = '?' - i++ - - runes[i] = ',' - i++ - } - - runes[0] = '(' - runes[len(runes)-1] = ')' - - return string(runes) -} diff --git a/mysql_utils.go b/mysql_utils.go new file mode 100644 index 00000000..38554e60 --- /dev/null +++ b/mysql_utils.go @@ -0,0 +1,367 @@ +package icingadb_connection + +import ( + "database/sql" + "encoding/hex" + "errors" + "fmt" + "github.com/go-sql-driver/mysql" + "io/ioutil" + "reflect" + "sort" + "strconv" + "strings" + log "github.com/sirupsen/logrus" + oldlog "log" +) + +// mkMysql creates a new MySQL client. +func mkMysql(dbType string, dbDsn string) (*sql.DB, error) { + log.Info("Connecting to MySQL") + + sep := "?" + + if dbDsn == "" { + dbDsn = "/" + } else { + dsnParts := strings.Split(dbDsn, "/") + if strings.Contains(dsnParts[len(dsnParts)-1], "?") { + sep = "&" + } + } + + dbDsn = dbDsn + sep + + "innodb_strict_mode=1&sql_mode='STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_ENGINE_SUBSTITUTION,PIPES_AS_CONCAT,ANSI_QUOTES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER'" + + db, errConn := sql.Open(dbType, dbDsn) + if errConn != nil { + return nil, errConn + } + + mysql.SetLogger(oldlog.New(ioutil.Discard, "", 0)) + + db.SetMaxOpenConns(100) + db.SetMaxIdleConns(0) + + return db, nil +} + +type dbTypeBridge interface { + sql.Scanner + Result() interface{} +} + +type dbIntBridge struct { + result interface{} +} + +func (d *dbIntBridge) Scan(src interface{}) (err error) { + baseScanner := sql.NullInt64{} + err = baseScanner.Scan(src) + + if err == nil { + if baseScanner.Valid { + d.result = baseScanner.Int64 + } else { + d.result = nil + } + } + + return +} + +func (d *dbIntBridge) Result() interface{} { + return d.result +} + +type dbFloatBridge struct { + result interface{} +} + +func (d *dbFloatBridge) Scan(src interface{}) (err error) { + baseScanner := sql.NullFloat64{} + err = baseScanner.Scan(src) + + if err == nil { + if baseScanner.Valid { + d.result = baseScanner.Float64 + } else { + d.result = nil + } + } + + return +} + +func (d *dbFloatBridge) Result() interface{} { + return d.result +} + +type dbStringBridge struct { + result interface{} +} + +func (d *dbStringBridge) Scan(src interface{}) (err error) { + baseScanner := sql.NullString{} + err = baseScanner.Scan(src) + + if err == nil { + if baseScanner.Valid { + d.result = baseScanner.String + } else { + d.result = nil + } + } + + return +} + +func (d *dbStringBridge) Result() interface{} { + return d.result +} + +type dbBytesBridge struct { + result interface{} +} + +func (d *dbBytesBridge) Scan(src interface{}) (err error) { + baseScanner := sql.NullString{} + err = baseScanner.Scan(src) + + if err == nil { + if baseScanner.Valid { + d.result = []byte(baseScanner.String) + } else { + d.result = nil + } + } + + return +} + +func (d *dbBytesBridge) Result() interface{} { + return d.result +} + +var dbTypeBridgeFactories = map[string]func() dbTypeBridge{ + // MySQL + "TINYINT": func() dbTypeBridge { + return &dbIntBridge{} + }, + "SMALLINT": func() dbTypeBridge { + return &dbIntBridge{} + }, + "INT": func() dbTypeBridge { + return &dbIntBridge{} + }, + "BIGINT": func() dbTypeBridge { + return &dbIntBridge{} + }, + "FLOAT": func() dbTypeBridge { + return &dbFloatBridge{} + }, + "CHAR": func() dbTypeBridge { + return &dbStringBridge{} + }, + "VARCHAR": func() dbTypeBridge { + return &dbStringBridge{} + }, + "ENUM": func() dbTypeBridge { + return &dbStringBridge{} + }, + "BINARY": func() dbTypeBridge { + return &dbBytesBridge{} + }, + + // SQLite + "INTEGER": func() dbTypeBridge { + return &dbIntBridge{} + }, + "REAL": func() dbTypeBridge { + return &dbFloatBridge{} + }, + "TEXT": func() dbTypeBridge { + return &dbStringBridge{} + }, + "BLOB": func() dbTypeBridge { + return &dbBytesBridge{} + }, + // SELECT 1 FROM ... + "": func() dbTypeBridge { + return &dbIntBridge{} + }, +} + +type dbBrokenBridge struct { + typ string +} + +func (d *dbBrokenBridge) Scan(src interface{}) error { + types := make([]string, len(dbTypeBridgeFactories)) + typeIdx := 0 + + for typ := range dbTypeBridgeFactories { + types[typeIdx] = typ + typeIdx++ + } + + sort.Strings(types) + + return errors.New(fmt.Sprintf("bad column type %s, expected one of %s", d.typ, strings.Join(types, ", "))) +} + +func (d *dbBrokenBridge) Result() interface{} { + return nil +} + +var prettyPrintedSqlReplacer = strings.NewReplacer("\n", " ", "\t", "") + +type prettyPrintedSql struct { + sql string +} + +// String implements and interface from Stringer +func (p prettyPrintedSql) String() string { + return strings.TrimSpace(prettyPrintedSqlReplacer.Replace(p.sql)) +} + +// MarshalText implements an interface from TextMarshaler +func (p prettyPrintedSql) MarshalText() (text []byte, err error) { + return []byte(p.String()), nil +} + +type prettyPrintedArgs struct { + args []interface{} +} + +func (p *prettyPrintedArgs) String() string { + res := "[" + + for _, v := range p.args { + if byteArray, isByteArray := v.([]byte); isByteArray { + res = fmt.Sprintf("%s hex.DecodeString(\"%s\"),", res, hex.EncodeToString(byteArray)) + } else { + res = fmt.Sprintf("%s %#v,", res, v) + } + } + + return res + " ]" +} + +// MarshalText implements an interface from TextMarshaler +func (p prettyPrintedArgs) MarshalText() (text []byte, err error) { + return []byte(p.String()), nil +} + +type prettyPrintedRowsAffected struct { + result sql.Result +} + +// String implements and interface from Stringer +func (d prettyPrintedRowsAffected) String() string { + if d.result != nil { + rows, errRA := d.result.RowsAffected() + if errRA == nil { + return strconv.FormatInt(rows, 10) + } + } + + return "N/A" +} + +// MarshalText implements an interface from TextMarshaler +func (d prettyPrintedRowsAffected) MarshalText() (text []byte, err error) { + return []byte(d.String()), nil +} + +type MysqlConnectionError struct { + err string +} + +func (e MysqlConnectionError) Error() string { + return e.err +} + +// Returns whether the given error signals serialization failure +// https://dev.mysql.com/doc/refman/5.5/en/error-messages-server.html#error_er_lock_deadlock +func isSerializationFailure(e error) bool { + switch err := e.(type) { + case *mysql.MySQLError: + switch err.Number { + // Those are the error numbers for serialization failures, upon which we retry + case 1205, 1213: + return true + } + } + + return false +} + +func formatLogQuery(query string) string { + r := strings.NewReplacer("\n", " ", "\t", "") + return strings.TrimSpace(r.Replace(query)) +} + +// Go bool -> DB bool +var yesNo = map[bool]string{ + true: "y", + false: "n", +} + +func ConvertValueForDb(in interface{}) interface{} { + switch value := in.(type) { + case []byte: + case string: + case float64: + case int64: + case nil: + break + case float32: + return float64(value) + case uint: + return int64(value) + case uint8: + return int64(value) + case uint16: + return int64(value) + case uint32: + return int64(value) + case uint64: + return int64(value) + case int: + return int64(value) + case int8: + return int64(value) + case int16: + return int64(value) + case int32: + return int64(value) + case bool: + return yesNo[value] + default: + panic(fmt.Sprintf( + "bad type %s, expected one of []byte, string, float{32,64}, {,u}int{,8,16,32,64}, bool, nil", + reflect.TypeOf(in).Name(), + )) + } + + return in +} + +func MakePlaceholderList(x int) string { + runes := make([]rune, 1+x*2) + + i := 1 + for j := 0; j < x; j++ { + runes[i] = '?' + i++ + + runes[i] = ',' + i++ + } + + runes[0] = '(' + runes[len(runes)-1] = ')' + + return string(runes) +} From f1d834e54be7233aa0c87147eb0a7578914e9fd7 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 20 Feb 2019 14:56:17 +0100 Subject: [PATCH 04/69] Make SqlRollback() public --- mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql.go b/mysql.go index c866a33a..95ddac16 100644 --- a/mysql.go +++ b/mysql.go @@ -184,7 +184,7 @@ func (dbw *DBWrapper) sqlTryTransaction(f func(*sql.Tx) error, concurrencySafety errTx := f(tx) if errTx != nil { - dbw.sqlRollback(tx, quiet) + dbw.SqlRollback(tx, quiet) return errTx } @@ -242,7 +242,7 @@ func (dbw *DBWrapper) SqlCommit(tx *sql.Tx, quiet bool) error { } // Wrapper around tx.Rollback() for auto-logging -func (dbw *DBWrapper) sqlRollback(tx *sql.Tx, quiet bool) error { +func (dbw *DBWrapper) SqlRollback(tx *sql.Tx, quiet bool) error { var err error if !quiet { benchmarc := benchmark.NewBenchmark() From 1c2d41f4470cf38d8b2228f4db89fa7a8b8b09bc Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 20 Feb 2019 16:26:17 +0100 Subject: [PATCH 05/69] Reimplement quiet functions --- mysql.go | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/mysql.go b/mysql.go index 95ddac16..dc6a1c68 100644 --- a/mysql.go +++ b/mysql.go @@ -191,6 +191,39 @@ func (dbw *DBWrapper) sqlTryTransaction(f func(*sql.Tx) error, concurrencySafety return dbw.SqlCommit(tx, quiet) } +// SqlTransaction executes the given function inside a transaction. +func (dbw *DBWrapper) SqlTransactionQuiet(concurrencySafety bool, retryOnConnectionFailure bool, f func(*sql.Tx) error) error { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + errTx := dbw.sqlTryTransaction(f, concurrencySafety, true) + if errTx != nil { + //TODO: Do this only for concurrencySafety = true, once we figure out the serialization errors. + if isSerializationFailure(errTx) { + continue + } + + if !dbw.checkConnection(false) { + if retryOnConnectionFailure { + continue + } else { + return MysqlConnectionError{"Transaction failed duo to a connection error"} + } + } + + // We still log errors + log.WithFields(log.Fields{ + "context": "sql", + "error": errTx, + }).Warn("SQL error occurred") + } + return errTx + } +} + // Wrapper around Db.BeginTx() for auto-logging func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (*sql.Tx, error) { var isoLvl sql.IsolationLevel @@ -363,6 +396,89 @@ func sqlTryFetchAll(db DbClient, queryDescription string, query string, args ... return res, nil } +// No logging, no benchmarking +func (dbw *DBWrapper) SqlFetchAllQuiet(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + res, err := sqlTryFetchAllQuiet(db, queryDescription, query, args...) + + if err != nil { + if _, isDb := db.(*sql.DB); isDb { + if !dbw.checkConnection(false) { + continue + } + } + } + + return res, err + } +} + +func sqlTryFetchAllQuiet(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + rows, errQuery := db.Query(query, args...) + + if errQuery != nil { + return [][]interface{}{}, errQuery + } + + defer rows.Close() + + columnTypes, errCT := rows.ColumnTypes() + if errCT != nil { + return [][]interface{}{}, errCT + } + + colsPerRow := len(columnTypes) + buf := list.New() + bridges := make([]dbTypeBridge, colsPerRow) + scanDest := make([]interface{}, colsPerRow) + + for i, columnType := range columnTypes { + typ := columnType.DatabaseTypeName() + factory, hasFactory := dbTypeBridgeFactories[typ] + if hasFactory { + bridges[i] = factory() + } else { + bridges[i] = &dbBrokenBridge{typ: typ} + } + + scanDest[i] = bridges[i] + } + + for { + if rows.Next() { + if errScan := rows.Scan(scanDest...); errScan != nil { + return [][]interface{}{}, errScan + } + + row := make([]interface{}, colsPerRow) + + for i, bridge := range bridges { + row[i] = bridge.Result() + } + + buf.PushBack(row) + } else if errNx := rows.Err(); errNx == nil { + break + } else { + return nil, errNx + } + } + + res := make([][]interface{}, buf.Len()) + + for current, i := buf.Front(), 0; current != nil; current = current.Next() { + res[i] = current.Value.([]interface{}) + i++ + } + + return res, nil +} + // Wrapper around tx.SqlExec() for auto-logging func (dbw *DBWrapper) SqlExec(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { benchmarc := benchmark.NewBenchmark() @@ -381,3 +497,9 @@ func (dbw *DBWrapper) SqlExec(tx *sql.Tx, opDescription string, sql string, args return res, err } + +// No logging, no benchmarking +func (dbw *DBWrapper) SqlExecQuiet(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { + res, err := tx.Exec(sql, args...) + return res, err +} From 07fbfd1f116ad869fc95abb2253fc903bf172d64 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 22 Feb 2019 09:24:36 +0100 Subject: [PATCH 06/69] Add SqlExec & SqlExecQuiet besides TX based execs --- mysql.go | 115 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 17 deletions(-) diff --git a/mysql.go b/mysql.go index dc6a1c68..05ef2736 100644 --- a/mysql.go +++ b/mysql.go @@ -479,27 +479,108 @@ func sqlTryFetchAllQuiet(db DbClient, queryDescription string, query string, arg return res, nil } -// Wrapper around tx.SqlExec() for auto-logging -func (dbw *DBWrapper) SqlExec(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { - benchmarc := benchmark.NewBenchmark() - res, err := tx.Exec(sql, args...) - benchmarc.Stop() +// Wrapper around tx.Exec() for auto-logging +func (dbw *DBWrapper) SqlExecTx(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } - //DbIoSeconds.WithLabelValues("mysql", opDescription).Observe(benchmarc.Seconds()) + benchmarc := benchmark.NewBenchmark() + res, err := tx.Exec(sql, args...) + benchmarc.Stop() - log.WithFields(log.Fields{ - "context": "sql", - "benchmark": benchmarc, - "affected_rows": prettyPrintedRowsAffected{res}, - "args": prettyPrintedArgs{args}, - "query": prettyPrintedSql{sql}, - }).Debug("Finished Exec") + //DbIoSeconds.WithLabelValues("mysql", opDescription).Observe(benchmarc.Seconds()) - return res, err + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + "affected_rows": prettyPrintedRowsAffected{res}, + "args": prettyPrintedArgs{args}, + "query": prettyPrintedSql{sql}, + }).Debug("Finished Exec") + + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + return res, err + } } // No logging, no benchmarking -func (dbw *DBWrapper) SqlExecQuiet(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { - res, err := tx.Exec(sql, args...) - return res, err +func (dbw *DBWrapper) SqlExecTxQuiet(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + res, err := tx.Exec(sql, args...) + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + return res, err + } } + +// Wrapper around sql.Exec() for auto-logging +func (dbw *DBWrapper) SqlExec(opDescription string, sql string, args ...interface{}) (sql.Result, error) { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + benchmarc := benchmark.NewBenchmark() + res, err := dbw.Db.Exec(sql, args...) + benchmarc.Stop() + + //DbIoSeconds.WithLabelValues("mysql", opDescription).Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + "affected_rows": prettyPrintedRowsAffected{res}, + "args": prettyPrintedArgs{args}, + "query": prettyPrintedSql{sql}, + }).Debug("Finished Exec") + + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + return res, err + } +} + +// No logging, no benchmarking +func (dbw *DBWrapper) SqlExecQuiet(opDescription string, sql string, args ...interface{}) (sql.Result, error) { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + res, err := dbw.Db.Exec(sql, args...) + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + return res, err + } +} \ No newline at end of file From e63e745aeb0218b8ed0f899e62f240c94e33a02a Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 22 Feb 2019 09:25:18 +0100 Subject: [PATCH 07/69] Use interface DBClient for better testability --- mysql.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mysql.go b/mysql.go index 05ef2736..5804d4ab 100644 --- a/mysql.go +++ b/mysql.go @@ -14,11 +14,14 @@ import ( // Either a connection or a transaction type DbClient interface { Query(query string, args ...interface{}) (*sql.Rows, error) + Ping() error + BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) + Exec(query string, args ...interface{}) (sql.Result, error) } // Database wrapper including helper functions type DBWrapper struct { - Db *sql.DB + Db DbClient ConnectedAtomic *uint32 //uint32 to be able to use atomic operations ConnectionUpCondition *sync.Cond ConnectionLostCounter int From 530999cc2a0ab8b99034caf410d952bebe01b295 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 22 Feb 2019 09:25:36 +0100 Subject: [PATCH 08/69] Add connection error handling to some functions --- mysql.go | 125 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 82 insertions(+), 43 deletions(-) diff --git a/mysql.go b/mysql.go index 5804d4ab..46c26b50 100644 --- a/mysql.go +++ b/mysql.go @@ -236,66 +236,105 @@ func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (*sql.Tx, err isoLvl = sql.LevelReadCommitted } - var err error - var tx *sql.Tx - if quiet { - tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) - } else { - benchmarc := benchmark.NewBenchmark() - tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) - benchmarc.Stop() + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } - //DbIoSeconds.WithLabelValues("mysql", "begin").Observe(benchmarc.Seconds()) + var err error + var tx *sql.Tx + if quiet { + tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) + } else { + benchmarc := benchmark.NewBenchmark() + tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) + benchmarc.Stop() - log.WithFields(log.Fields{ - "context": "sql", - "benchmark": benchmarc, - }).Debug("BEGIN transaction") + //DbIoSeconds.WithLabelValues("mysql", "begin").Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + }).Debug("BEGIN transaction") + } + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + return tx, err } - - return tx, err } // Wrapper around tx.Commit() for auto-logging func (dbw *DBWrapper) SqlCommit(tx *sql.Tx, quiet bool) error { - var err error - if quiet { - err = tx.Commit() - } else { - benchmarc := benchmark.NewBenchmark() - err = tx.Commit() - benchmarc.Stop() + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } - //DbIoSeconds.WithLabelValues("mysql", "commit").Observe(benchmarc.Seconds()) + var err error + if quiet { + err = tx.Commit() + } else { + benchmarc := benchmark.NewBenchmark() + err = tx.Commit() + benchmarc.Stop() - log.WithFields(log.Fields{ - "context": "sql", - "benchmark": benchmarc, - }).Debug("COMMIT transaction") + //DbIoSeconds.WithLabelValues("mysql", "commit").Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + }).Debug("COMMIT transaction") + } + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + return err } - - return err } // Wrapper around tx.Rollback() for auto-logging func (dbw *DBWrapper) SqlRollback(tx *sql.Tx, quiet bool) error { - var err error - if !quiet { - benchmarc := benchmark.NewBenchmark() - err = tx.Rollback() - benchmarc.Stop() + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } - //DbIoSeconds.WithLabelValues("mysql", "rollback").Observe(benchmarc.Seconds()) + var err error + if !quiet { + benchmarc := benchmark.NewBenchmark() + err = tx.Rollback() + benchmarc.Stop() - log.WithFields(log.Fields{ - "context": "sql", - "benchmark": benchmarc, - }).Debug("ROLLBACK transaction") - } else { - err = tx.Rollback() + //DbIoSeconds.WithLabelValues("mysql", "rollback").Observe(benchmarc.Seconds()) + + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + }).Debug("ROLLBACK transaction") + } else { + err = tx.Rollback() + } + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + return err } - - return err } // Wrapper around Db.SqlQuery() for auto-logging From c182cb713a507e108a79ef214b530428a15926e6 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 08:48:08 +0100 Subject: [PATCH 09/69] Don't panic in ConvertValueForDb() --- mysql_utils.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/mysql_utils.go b/mysql_utils.go index 38554e60..2638dd3e 100644 --- a/mysql_utils.go +++ b/mysql_utils.go @@ -308,44 +308,43 @@ var yesNo = map[bool]string{ false: "n", } -func ConvertValueForDb(in interface{}) interface{} { +func ConvertValueForDb(in interface{}) (interface{}, error) { switch value := in.(type) { case []byte: case string: case float64: case int64: case nil: - break case float32: - return float64(value) + return float64(value), nil case uint: - return int64(value) + return int64(value), nil case uint8: - return int64(value) + return int64(value), nil case uint16: - return int64(value) + return int64(value), nil case uint32: - return int64(value) + return int64(value), nil case uint64: - return int64(value) + return int64(value), nil case int: - return int64(value) + return int64(value), nil case int8: - return int64(value) + return int64(value), nil case int16: - return int64(value) + return int64(value), nil case int32: - return int64(value) + return int64(value), nil case bool: - return yesNo[value] + return yesNo[value], nil default: - panic(fmt.Sprintf( + return nil, errors.New(fmt.Sprintf( "bad type %s, expected one of []byte, string, float{32,64}, {,u}int{,8,16,32,64}, bool, nil", reflect.TypeOf(in).Name(), )) } - return in + return in, nil } func MakePlaceholderList(x int) string { From b6c92fc88375b94d6fe983b7151f4a0413725b30 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 09:25:49 +0100 Subject: [PATCH 10/69] Implement DbTransaction interface to increase testability --- mysql.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/mysql.go b/mysql.go index 46c26b50..f063bdfd 100644 --- a/mysql.go +++ b/mysql.go @@ -6,12 +6,12 @@ import ( "database/sql" "git.icinga.com/icingadb/icingadb/benchmark" log "github.com/sirupsen/logrus" + "strings" "sync" "sync/atomic" "time" ) -// Either a connection or a transaction type DbClient interface { Query(query string, args ...interface{}) (*sql.Rows, error) Ping() error @@ -19,6 +19,13 @@ type DbClient interface { Exec(query string, args ...interface{}) (sql.Result, error) } +type DbTransaction interface { + Query(query string, args ...interface{}) (*sql.Rows, error) + Exec(query string, args ...interface{}) (sql.Result, error) + Commit() error + Rollback() error +} + // Database wrapper including helper functions type DBWrapper struct { Db DbClient @@ -77,7 +84,7 @@ func (dbw *DBWrapper) getConnectionCheckInterval() time.Duration { return 15 * time.Second } -func (dbw *DBWrapper) Query(query string, args ...interface{}) (*sql.Rows, error) { +func (dbw *DBWrapper) SqlQuery(query string, args ...interface{}) (*sql.Rows, error) { for { if !dbw.IsConnected() { dbw.WaitForConnection() @@ -270,7 +277,7 @@ func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (*sql.Tx, err } // Wrapper around tx.Commit() for auto-logging -func (dbw *DBWrapper) SqlCommit(tx *sql.Tx, quiet bool) error { +func (dbw *DBWrapper) SqlCommit(tx DbTransaction, quiet bool) error { for { if !dbw.IsConnected() { dbw.WaitForConnection() @@ -304,7 +311,7 @@ func (dbw *DBWrapper) SqlCommit(tx *sql.Tx, quiet bool) error { } // Wrapper around tx.Rollback() for auto-logging -func (dbw *DBWrapper) SqlRollback(tx *sql.Tx, quiet bool) error { +func (dbw *DBWrapper) SqlRollback(tx DbTransaction, quiet bool) error { for { if !dbw.IsConnected() { dbw.WaitForConnection() @@ -522,7 +529,7 @@ func sqlTryFetchAllQuiet(db DbClient, queryDescription string, query string, arg } // Wrapper around tx.Exec() for auto-logging -func (dbw *DBWrapper) SqlExecTx(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { +func (dbw *DBWrapper) SqlExecTx(tx DbTransaction, opDescription string, sql string, args ...interface{}) (sql.Result, error) { for { if !dbw.IsConnected() { dbw.WaitForConnection() @@ -555,7 +562,7 @@ func (dbw *DBWrapper) SqlExecTx(tx *sql.Tx, opDescription string, sql string, ar } // No logging, no benchmarking -func (dbw *DBWrapper) SqlExecTxQuiet(tx *sql.Tx, opDescription string, sql string, args ...interface{}) (sql.Result, error) { +func (dbw *DBWrapper) SqlExecTxQuiet(tx DbTransaction, opDescription string, sql string, args ...interface{}) (sql.Result, error) { for { if !dbw.IsConnected() { dbw.WaitForConnection() From 7bcba36c6eeb576a32e8d032c5df024adea45a47 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 09:54:03 +0100 Subject: [PATCH 11/69] Add Redis PubSub --- redis.go | 6 +++++ redis_pubsub.go | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 redis_pubsub.go diff --git a/redis.go b/redis.go index 3aa42c5b..a3540ea1 100644 --- a/redis.go +++ b/redis.go @@ -272,3 +272,9 @@ func (rdbw *RDBWrapper) TxPipelined(fn func(pipeliner redis.Pipeliner) error) ([ return c, e } } + +func (rdbw *RDBWrapper) Subscribe() PubSubWrapper { + ps := rdbw.Rdb.Subscribe() + psw := PubSubWrapper{ps: ps, rdbw: rdbw} + return psw +} \ No newline at end of file diff --git a/redis_pubsub.go b/redis_pubsub.go new file mode 100644 index 00000000..1cedb303 --- /dev/null +++ b/redis_pubsub.go @@ -0,0 +1,67 @@ +package icingadb_connection + +import ( + "github.com/go-redis/redis" +) + +type PubSubWrapper struct { + ps *redis.PubSub + rdbw *RDBWrapper +} + +func (psw *PubSubWrapper) Subscribe(channels ...string) error { + for { + if !psw.rdbw.IsConnected() { + psw.rdbw.WaitForConnection() + continue + } + + err := psw.ps.Subscribe(channels...) + + if err != nil { + if !psw.rdbw.CheckConnection(false) { + continue + } + } + + return err + } +} + +func (psw *PubSubWrapper) ReceiveMessage() (*redis.Message, error) { + for { + if !psw.rdbw.IsConnected() { + psw.rdbw.WaitForConnection() + continue + } + + msg, err := psw.ps.ReceiveMessage() + + if err != nil { + if !psw.rdbw.CheckConnection(false) { + continue + } + } + + return msg, err + } +} + +func (psw *PubSubWrapper) Close() error { + for { + if !psw.rdbw.IsConnected() { + psw.rdbw.WaitForConnection() + continue + } + + err := psw.ps.Close() + + if err != nil { + if !psw.rdbw.CheckConnection(false) { + continue + } + } + + return err + } +} \ No newline at end of file From b3afe0385ad2fd5cf876b8be002f0bd47eaf2578 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 10:33:17 +0100 Subject: [PATCH 12/69] DBWrapper: Test connection in NewDBWrapper() --- mysql.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mysql.go b/mysql.go index f063bdfd..96298c4e 100644 --- a/mysql.go +++ b/mysql.go @@ -56,6 +56,11 @@ func NewDBWrapper(dbType string, dbDsn string) (*DBWrapper, error) { dbw := DBWrapper{Db: db, ConnectedAtomic: new(uint32)} dbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) + err = dbw.Db.Ping() + if err != nil { + return nil, err + } + go func() { for { dbw.checkConnection(true) From daff13165d6f6095c034da5ba2e0ac690dee04f9 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 10:33:39 +0100 Subject: [PATCH 13/69] Add WithRetry() to DBWrapper --- mysql.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mysql.go b/mysql.go index 96298c4e..81bdffc7 100644 --- a/mysql.go +++ b/mysql.go @@ -635,6 +635,29 @@ func (dbw *DBWrapper) SqlExecQuiet(opDescription string, sql string, args ...int } } + return res, err + } +} + +func IsRetryableError(err error) bool { + if strings.Contains(err.Error(), "Deadlock found when trying to get lock") { + return true + } + return false +} + +func (dbw *DBWrapper) WithRetry(f func() (sql.Result, error)) (sql.Result, error) { + for { + res, err := f() + + if err != nil { + if IsRetryableError(err) { + continue + } else { + return nil, err + } + } + return res, err } } \ No newline at end of file From fe7e0beb014ed418d39c342d049488d11b69e4a0 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 10:35:00 +0100 Subject: [PATCH 14/69] Add first tests --- mysql_test.go | 295 ++++++++++++++++++++++++++++++++++++++++++++ mysql_utils_test.go | 104 ++++++++++++++++ 2 files changed, 399 insertions(+) create mode 100644 mysql_test.go create mode 100644 mysql_utils_test.go diff --git a/mysql_test.go b/mysql_test.go new file mode 100644 index 00000000..e42340b7 --- /dev/null +++ b/mysql_test.go @@ -0,0 +1,295 @@ +package icingadb_connection + +import ( + "context" + "database/sql" + "errors" + "github.com/go-sql-driver/mysql" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "sync" + "testing" + "time" +) + +type SqlResultMock struct { + sql.Result +} +type TransactionMock struct { + mock.Mock +} + +func (m *TransactionMock) Query(query string, args ...interface{}) (*sql.Rows, error) { + args2 := m.Called(query, args) + return args2.Get(0).(*sql.Rows), args2.Error(1) +} + +func (m *TransactionMock) Exec(query string, args ...interface{}) (sql.Result, error) { + args2 := m.Called(query, args) + return args2.Get(0).(sql.Result), args2.Error(1) +} + +func (m *TransactionMock) Commit() error { + args := m.Called() + return args.Error(0) +} + +func (m *TransactionMock) Rollback() error { + args := m.Called() + return args.Error(0) +} + +type DbMock struct { + mock.Mock +} + +func (m *DbMock) Ping() error { + args := m.Called() + return args.Error(0) +} + +func (m *DbMock) Query(query string, args ...interface{}) (*sql.Rows, error) { + args2 := m.Called(query, args) + return args2.Get(0).(*sql.Rows), args2.Error(1) +} + +func (m *DbMock) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) { + args := m.Called(ctx, opts) + return args.Get(0).(*sql.Tx), args.Error(1) +} + +func (m *DbMock) Exec(query string, args ...interface{}) (sql.Result, error) { + args2 := m.Called(query, args) + return args2.Get(0).(sql.Result), args2.Error(1) +} + +func NewTestDBW(db DbClient) DBWrapper { + dbw := DBWrapper{Db: db, ConnectedAtomic: new(uint32)} + dbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) + return dbw +} + +func TestNewDBWrapper(t *testing.T) { + _, err := NewDBWrapper("mysql", "icingadb:icingadb@tcp(127.0.0.1:3306)/icingadb") + if err != nil { + logrus.Error(err.Error()) + } + assert.Nil(t, err) +} + +func TestRDBWrapper_CheckConnection(t *testing.T) { + mockDb := new(DbMock) + dbw := NewTestDBW(mockDb) + + dbw.ConnectionLostCounter = 180239812 + mockDb.On("Ping").Return(nil).Once() + assert.True(t, dbw.checkConnection(false), "DBWrapper should be connected") + assert.Equal(t, 0, dbw.ConnectionLostCounter) + + dbw.ConnectionLostCounter = 0 + mockDb.On("Ping").Return(mysql.ErrInvalidConn).Once() + assert.False(t, dbw.checkConnection(false), "DBWrapper should not be connected") + assert.Equal(t, 0, dbw.ConnectionLostCounter) + + dbw.ConnectionLostCounter = 10 + mockDb.On("Ping").Return(mysql.ErrInvalidConn).Once() + assert.False(t, dbw.checkConnection(true), "DBWrapper should not be connected") + assert.Equal(t, 11, dbw.ConnectionLostCounter) +} + +func TestDBWrapper_WithRetry(t *testing.T) { + mockDb := new(DbMock) + dbw := NewTestDBW(mockDb) + + tries := 0 + + _, err := dbw.WithRetry(func() (result sql.Result, e error) { + if tries > 0 { + tries++ + return nil, nil + } else { + tries++ + return nil, errors.New("Deadlock found when trying to get lock") + } + }) + + assert.Nil(t, err) + assert.Equal(t, 2, tries) +} + +func TestDBWrapper_SqlQuery(t *testing.T) { + mockDb := new(DbMock) + dbw := NewTestDBW(mockDb) + + mockDb.On("Query", "test", []interface{}(nil)).Return(&sql.Rows{}, errors.New("whoops")).Once() + mockDb.On("Query", "test", []interface{}(nil)).Return(&sql.Rows{}, nil).Once() + mockDb.On("Ping").Return(errors.New("whoops")).Once() + + var err error + done := make(chan bool) + + dbw.CompareAndSetConnected(true) + go func() { + _, err = dbw.SqlQuery("test") + done <- true + }() + + time.Sleep(time.Millisecond * 100) + + dbw.CompareAndSetConnected(true) + dbw.ConnectionUpCondition.Broadcast() + + <- done + + assert.Nil(t, err) + mockDb.AssertExpectations(t) +} + +func TestDBWrapper_SqlExec(t *testing.T) { + mockDb := new(DbMock) + dbw := NewTestDBW(mockDb) + + mockDb.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, errors.New("whoops")).Once() + mockDb.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, nil).Once() + mockDb.On("Ping").Return(errors.New("whoops")).Once() + + var err error + done := make(chan bool) + + dbw.CompareAndSetConnected(true) + go func() { + _, err = dbw.SqlExec("test", "test") + done <- true + }() + + time.Sleep(time.Millisecond * 100) + + dbw.CompareAndSetConnected(true) + dbw.ConnectionUpCondition.Broadcast() + + <- done + + assert.Nil(t, err) + mockDb.AssertExpectations(t) +} + +func TestDBWrapper_SqlExecQuiet(t *testing.T) { + mockDb := new(DbMock) + dbw := NewTestDBW(mockDb) + + mockDb.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, errors.New("whoops")).Once() + mockDb.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, nil).Once() + mockDb.On("Ping").Return(errors.New("whoops")).Once() + + var err error + done := make(chan bool) + + dbw.CompareAndSetConnected(true) + go func() { + _, err = dbw.SqlExecQuiet("test", "test") + done <- true + }() + + time.Sleep(time.Millisecond * 100) + + dbw.CompareAndSetConnected(true) + dbw.ConnectionUpCondition.Broadcast() + + <- done + + assert.Nil(t, err) + mockDb.AssertExpectations(t) +} + +func TestDBWrapper_SqlExecTx(t *testing.T) { + mockDb := new(DbMock) + dbw := NewTestDBW(mockDb) + mockTx := new(TransactionMock) + + mockTx.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, errors.New("whoops")).Once() + mockTx.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, nil).Once() + mockDb.On("Ping").Return(errors.New("whoops")).Once() + + var err error + done := make(chan bool) + + dbw.CompareAndSetConnected(true) + go func() { + _, err = dbw.SqlExecTx(mockTx, "test", "test") + done <- true + }() + + time.Sleep(time.Millisecond * 100) + + dbw.CompareAndSetConnected(true) + dbw.ConnectionUpCondition.Broadcast() + + <- done + + assert.Nil(t, err) + mockTx.AssertExpectations(t) + mockDb.AssertExpectations(t) +} + +func TestDBWrapper_SqlExecTxQuiet(t *testing.T) { + mockDb := new(DbMock) + dbw := NewTestDBW(mockDb) + mockTx := new(TransactionMock) + + mockTx.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, errors.New("whoops")).Once() + mockTx.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, nil).Once() + mockDb.On("Ping").Return(errors.New("whoops")).Once() + + var err error + done := make(chan bool) + + dbw.CompareAndSetConnected(true) + go func() { + _, err = dbw.SqlExecTxQuiet(mockTx, "test", "test") + done <- true + }() + + time.Sleep(time.Millisecond * 100) + + dbw.CompareAndSetConnected(true) + dbw.ConnectionUpCondition.Broadcast() + + <- done + + assert.Nil(t, err) + mockTx.AssertExpectations(t) + mockDb.AssertExpectations(t) +} + +func TestGetConnectionCheckInterval(t *testing.T) { + dbw := NewTestDBW(nil) + + //Should return 15s, if connected - counter doesn't madder + dbw.CompareAndSetConnected(true) + assert.Equal(t, 15*time.Second, dbw.getConnectionCheckInterval()) + + //Should return 5s, if not connected and counter < 4 + dbw.CompareAndSetConnected(false) + dbw.ConnectionLostCounter = 0 + assert.Equal(t, 5*time.Second, dbw.getConnectionCheckInterval()) + + //Should return 10s, if not connected and 4 <= counter < 8 + dbw.CompareAndSetConnected(false) + dbw.ConnectionLostCounter = 4 + assert.Equal(t, 10*time.Second, dbw.getConnectionCheckInterval()) + + //Should return 30s, if not connected and 8 <= counter < 11 + dbw.CompareAndSetConnected(false) + dbw.ConnectionLostCounter = 8 + assert.Equal(t, 30*time.Second, dbw.getConnectionCheckInterval()) + + //Should return 60s, if not connected and 11 <= counter < 14 + dbw.CompareAndSetConnected(false) + dbw.ConnectionLostCounter = 11 + assert.Equal(t, 60*time.Second, dbw.getConnectionCheckInterval()) + + //dbw.ConnectionLostCounter = 14 + //interval = dbw.getConnectionCheckInterval() + //TODO: Check for Fatal +} diff --git a/mysql_utils_test.go b/mysql_utils_test.go new file mode 100644 index 00000000..b21bc171 --- /dev/null +++ b/mysql_utils_test.go @@ -0,0 +1,104 @@ +package icingadb_connection + +import ( + "errors" + "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestMakePlaceholderList(t *testing.T) { + assert.Equal(t, "(?)", MakePlaceholderList(1)) + assert.Equal(t, "(?,?,?,?,?)", MakePlaceholderList(5)) + assert.Equal(t, "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", MakePlaceholderList(20)) +} + +func TestConvertValueForDb(t *testing.T) { + var v interface{} + var err error + + v, err = ConvertValueForDb(nil) + assert.IsType(t, nil, v) + assert.Nil(t, err) + + v, err = ConvertValueForDb([]byte{100}) + assert.IsType(t, []byte{100}, v) + assert.Nil(t, err) + + v, err = ConvertValueForDb("this-is-a-string") + assert.IsType(t, "this-is-a-string", v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(float32(123.456)) + assert.IsType(t, float64(123.456), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(float64(123.456)) + assert.IsType(t, float64(123.456), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(uint(20)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(uint8(30)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(uint16(40)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(uint32(50)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(uint64(60)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(int(70)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(int8(80)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(int16(90)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(int32(100)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(int64(10)) + assert.IsType(t, int64(10), v) + assert.Nil(t, err) + + v, err = ConvertValueForDb(true) + assert.IsType(t, "y/n-string", v) + assert.Nil(t, err) + + //Should not be possible + v, err = ConvertValueForDb(errors.New("test")) + assert.NotNil(t, err) +} + +func TestIsSerializationFailure(t *testing.T) { + assert.True(t, isSerializationFailure(&mysql.MySQLError{Number: 1205})) + assert.True(t, isSerializationFailure(&mysql.MySQLError{Number: 1213})) + + assert.False(t, isSerializationFailure(&mysql.MySQLError{Number: 6342})) + assert.False(t, isSerializationFailure(errors.New("random error"))) +} + +func TestMysqlConnectionError_Error(t *testing.T) { + err := MysqlConnectionError{"The chicken has left the database!"} + assert.Equal(t, "The chicken has left the database!", err.Error()) +} + +func TestFormatLogQuery(t *testing.T) { + assert.Equal(t, "This is my string", formatLogQuery("\tThis is\nmy string\n")) +} \ No newline at end of file From 3c7445e4c98f74f4977e335e4cc27933d6f9a091 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 11:08:06 +0100 Subject: [PATCH 15/69] Use icingadb-utils-lib for benchmarks --- mysql.go | 16 ++++++++-------- redis.go | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mysql.go b/mysql.go index 81bdffc7..40202e12 100644 --- a/mysql.go +++ b/mysql.go @@ -4,7 +4,7 @@ import ( "container/list" "context" "database/sql" - "git.icinga.com/icingadb/icingadb/benchmark" + "git.icinga.com/icingadb/icingadb-utils-lib" log "github.com/sirupsen/logrus" "strings" "sync" @@ -151,7 +151,7 @@ func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFai continue } - benchmarc := benchmark.NewBenchmark() + benchmarc := icingadb_utils.NewBenchmark() errTx := dbw.sqlTryTransaction(f, concurrencySafety, false) benchmarc.Stop() @@ -259,7 +259,7 @@ func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (*sql.Tx, err if quiet { tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) } else { - benchmarc := benchmark.NewBenchmark() + benchmarc := icingadb_utils.NewBenchmark() tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) benchmarc.Stop() @@ -293,7 +293,7 @@ func (dbw *DBWrapper) SqlCommit(tx DbTransaction, quiet bool) error { if quiet { err = tx.Commit() } else { - benchmarc := benchmark.NewBenchmark() + benchmarc := icingadb_utils.NewBenchmark() err = tx.Commit() benchmarc.Stop() @@ -325,7 +325,7 @@ func (dbw *DBWrapper) SqlRollback(tx DbTransaction, quiet bool) error { var err error if !quiet { - benchmarc := benchmark.NewBenchmark() + benchmarc := icingadb_utils.NewBenchmark() err = tx.Rollback() benchmarc.Stop() @@ -372,7 +372,7 @@ func (dbw *DBWrapper) SqlFetchAll(db DbClient, queryDescription string, query st } func sqlTryFetchAll(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { - benchmarc := benchmark.NewBenchmark() + benchmarc := icingadb_utils.NewBenchmark() rows, errQuery := db.Query(query, args...) benchmarc.Stop() @@ -541,7 +541,7 @@ func (dbw *DBWrapper) SqlExecTx(tx DbTransaction, opDescription string, sql stri continue } - benchmarc := benchmark.NewBenchmark() + benchmarc := icingadb_utils.NewBenchmark() res, err := tx.Exec(sql, args...) benchmarc.Stop() @@ -594,7 +594,7 @@ func (dbw *DBWrapper) SqlExec(opDescription string, sql string, args ...interfac continue } - benchmarc := benchmark.NewBenchmark() + benchmarc := icingadb_utils.NewBenchmark() res, err := dbw.Db.Exec(sql, args...) benchmarc.Stop() diff --git a/redis.go b/redis.go index a3540ea1..d28a371d 100644 --- a/redis.go +++ b/redis.go @@ -1,7 +1,7 @@ package icingadb_connection import ( - "git.icinga.com/icingadb/icingadb/benchmark" + "git.icinga.com/icingadb/icingadb-utils-lib" "github.com/go-redis/redis" log "github.com/sirupsen/logrus" "sync" @@ -219,7 +219,7 @@ func (rdbw *RDBWrapper) HGetAll(key string) (map[string]string, error) { continue } - benchmarc := benchmark.NewBenchmark() + benchmarc := icingadb_utils.NewBenchmark() res, errHGA := rdbw.Rdb.HGetAll(key).Result() if errHGA != nil { @@ -248,9 +248,9 @@ func (rdbw *RDBWrapper) TxPipelined(fn func(pipeliner redis.Pipeliner) error) ([ for { if !rdbw.IsConnected() { rdbw.WaitForConnection() - continue + continue } - benchmarc := benchmark.NewBenchmark() + benchmarc := icingadb_utils.NewBenchmark() c, e := rdbw.Rdb.TxPipelined(fn) if e != nil { From 233decef22a79d47c4943c3696cc16985ec35783 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 13:02:11 +0100 Subject: [PATCH 16/69] Temp fix for TestNewDBWrapper() --- mysql_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mysql_test.go b/mysql_test.go index e42340b7..4b355d43 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -5,7 +5,6 @@ import ( "database/sql" "errors" "github.com/go-sql-driver/mysql" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "sync" @@ -71,11 +70,9 @@ func NewTestDBW(db DbClient) DBWrapper { } func TestNewDBWrapper(t *testing.T) { - _, err := NewDBWrapper("mysql", "icingadb:icingadb@tcp(127.0.0.1:3306)/icingadb") - if err != nil { - logrus.Error(err.Error()) - } - assert.Nil(t, err) + _, err := NewDBWrapper("mysql", "asdasd") + assert.NotNil(t, err) + //TODO: Add more tests here } func TestRDBWrapper_CheckConnection(t *testing.T) { From c99ea629878bf15cb9b7fdb18d68bd45851b490d Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 13:33:12 +0100 Subject: [PATCH 17/69] Fix race condition --- mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql.go b/mysql.go index 40202e12..6d9baf7d 100644 --- a/mysql.go +++ b/mysql.go @@ -35,7 +35,7 @@ type DBWrapper struct { } func (dbw *DBWrapper) IsConnected() bool { - return *dbw.ConnectedAtomic != 0 + return atomic.LoadUint32(dbw.ConnectedAtomic) != 0 } func (dbw *DBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { From b658a6284596ff8dec9a8d26b708f0ca67e448ab Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 14:28:09 +0100 Subject: [PATCH 18/69] Add DbClientOrTransaction interface to be able to use FetchAll on Tx and Db --- mysql.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mysql.go b/mysql.go index 6d9baf7d..a484ddbf 100644 --- a/mysql.go +++ b/mysql.go @@ -26,6 +26,12 @@ type DbTransaction interface { Rollback() error } +// This is used in SqlFetchAll and SqlFetchAllQuiet +type DbClientOrTransaction interface { + Query(query string, args ...interface{}) (*sql.Rows, error) + Exec(query string, args ...interface{}) (sql.Result, error) +} + // Database wrapper including helper functions type DBWrapper struct { Db DbClient @@ -350,7 +356,7 @@ func (dbw *DBWrapper) SqlRollback(tx DbTransaction, quiet bool) error { } // Wrapper around Db.SqlQuery() for auto-logging -func (dbw *DBWrapper) SqlFetchAll(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { +func (dbw *DBWrapper) SqlFetchAll(db DbClientOrTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { for { if !dbw.IsConnected() { dbw.WaitForConnection() @@ -371,7 +377,7 @@ func (dbw *DBWrapper) SqlFetchAll(db DbClient, queryDescription string, query st } } -func sqlTryFetchAll(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { +func sqlTryFetchAll(db DbClientOrTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { benchmarc := icingadb_utils.NewBenchmark() rows, errQuery := db.Query(query, args...) benchmarc.Stop() @@ -451,7 +457,7 @@ func sqlTryFetchAll(db DbClient, queryDescription string, query string, args ... } // No logging, no benchmarking -func (dbw *DBWrapper) SqlFetchAllQuiet(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { +func (dbw *DBWrapper) SqlFetchAllQuiet(db DbClientOrTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { for { if !dbw.IsConnected() { dbw.WaitForConnection() @@ -472,7 +478,7 @@ func (dbw *DBWrapper) SqlFetchAllQuiet(db DbClient, queryDescription string, que } } -func sqlTryFetchAllQuiet(db DbClient, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { +func sqlTryFetchAllQuiet(db DbClientOrTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { rows, errQuery := db.Query(query, args...) if errQuery != nil { From 19b81b499ef5dbe1060d3ce45c39b869028d1dd6 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 25 Feb 2019 16:16:29 +0100 Subject: [PATCH 19/69] Simplify interfaces --- mysql.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/mysql.go b/mysql.go index a484ddbf..b0c451bc 100644 --- a/mysql.go +++ b/mysql.go @@ -12,26 +12,24 @@ import ( "time" ) -type DbClient interface { - Query(query string, args ...interface{}) (*sql.Rows, error) - Ping() error - BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) - Exec(query string, args ...interface{}) (sql.Result, error) -} - -type DbTransaction interface { - Query(query string, args ...interface{}) (*sql.Rows, error) - Exec(query string, args ...interface{}) (sql.Result, error) - Commit() error - Rollback() error -} - // This is used in SqlFetchAll and SqlFetchAllQuiet type DbClientOrTransaction interface { Query(query string, args ...interface{}) (*sql.Rows, error) Exec(query string, args ...interface{}) (sql.Result, error) } +type DbClient interface { + DbClientOrTransaction + Ping() error + BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) +} + +type DbTransaction interface { + DbClientOrTransaction + Commit() error + Rollback() error +} + // Database wrapper including helper functions type DBWrapper struct { Db DbClient From 53486bfdb0f1889c0973bd18893058585739beb7 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 09:26:19 +0100 Subject: [PATCH 20/69] Refactor MySQL functions that use Tx/DB and Quiet/Non-Quiet --- mysql.go | 516 ++++++++++++++++++------------------------------- mysql_utils.go | 7 + 2 files changed, 195 insertions(+), 328 deletions(-) diff --git a/mysql.go b/mysql.go index b0c451bc..a3df3eaf 100644 --- a/mysql.go +++ b/mysql.go @@ -6,7 +6,6 @@ import ( "database/sql" "git.icinga.com/icingadb/icingadb-utils-lib" log "github.com/sirupsen/logrus" - "strings" "sync" "sync/atomic" "time" @@ -30,26 +29,6 @@ type DbTransaction interface { Rollback() error } -// Database wrapper including helper functions -type DBWrapper struct { - Db DbClient - ConnectedAtomic *uint32 //uint32 to be able to use atomic operations - ConnectionUpCondition *sync.Cond - ConnectionLostCounter int -} - -func (dbw *DBWrapper) IsConnected() bool { - return atomic.LoadUint32(dbw.ConnectedAtomic) != 0 -} - -func (dbw *DBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { - if connected { - return atomic.CompareAndSwapUint32(dbw.ConnectedAtomic, 0, 1) - } else { - return atomic.CompareAndSwapUint32(dbw.ConnectedAtomic, 1, 0) - } -} - func NewDBWrapper(dbType string, dbDsn string) (*DBWrapper, error) { db, err := mkMysql(dbType, dbDsn) @@ -75,6 +54,26 @@ func NewDBWrapper(dbType string, dbDsn string) (*DBWrapper, error) { return &dbw, nil } +// Database wrapper including helper functions +type DBWrapper struct { + Db DbClient + ConnectedAtomic *uint32 //uint32 to be able to use atomic operations + ConnectionUpCondition *sync.Cond + ConnectionLostCounter int +} + +func (dbw *DBWrapper) IsConnected() bool { + return atomic.LoadUint32(dbw.ConnectedAtomic) != 0 +} + +func (dbw *DBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { + if connected { + return atomic.CompareAndSwapUint32(dbw.ConnectedAtomic, 0, 1) + } else { + return atomic.CompareAndSwapUint32(dbw.ConnectedAtomic, 1, 0) + } +} + func (dbw *DBWrapper) getConnectionCheckInterval() time.Duration { if !dbw.IsConnected() { if dbw.ConnectionLostCounter < 4 { @@ -93,25 +92,6 @@ func (dbw *DBWrapper) getConnectionCheckInterval() time.Duration { return 15 * time.Second } -func (dbw *DBWrapper) SqlQuery(query string, args ...interface{}) (*sql.Rows, error) { - for { - if !dbw.IsConnected() { - dbw.WaitForConnection() - continue - } - - res, err := dbw.Db.Query(query, args...) - - if err != nil { - if !dbw.checkConnection(false) { - continue - } - } - - return res, err - } -} - func (dbw *DBWrapper) checkConnection(isTicker bool) bool { err := dbw.Db.Ping() if err != nil { @@ -147,99 +127,38 @@ func (dbw *DBWrapper) WaitForConnection() { dbw.ConnectionUpCondition.L.Unlock() } -// SqlTransaction executes the given function inside a transaction. -func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFailure bool, f func(*sql.Tx) error) error { +func (dbw *DBWrapper) WithRetry(f func() (sql.Result, error)) (sql.Result, error) { + for { + res, err := f() + + if err != nil { + if isRetryableError(err) { + continue + } else { + return nil, err + } + } + + return res, err + } +} + +func (dbw *DBWrapper) SqlQuery(query string, args ...interface{}) (*sql.Rows, error) { for { if !dbw.IsConnected() { dbw.WaitForConnection() continue } - benchmarc := icingadb_utils.NewBenchmark() - errTx := dbw.sqlTryTransaction(f, concurrencySafety, false) - benchmarc.Stop() + res, err := dbw.Db.Query(query, args...) - //DbIoSeconds.WithLabelValues("mysql", "transaction").Observe(benchmarc.Seconds()) - - log.WithFields(log.Fields{ - "context": "sql", - "benchmark": benchmarc, - }).Debug("Executed transaction") - - if errTx != nil { - //TODO: Do this only for concurrencySafety = true, once we figure out the serialization errors. - if isSerializationFailure(errTx) { - log.WithFields(log.Fields{ - "context": "sql", - "error": errTx, - }).Debug("Repeating transaction") + if err != nil { + if !dbw.checkConnection(false) { continue } - - if !dbw.checkConnection(false) { - if retryOnConnectionFailure { - continue - } else { - return MysqlConnectionError{"Transaction failed duo to a connection error"} - } - } - - log.WithFields(log.Fields{ - "context": "sql", - "error": errTx, - }).Warn("SQL error occurred") } - return errTx - } -} - -// Executes the given function inside a transaction -func (dbw *DBWrapper) sqlTryTransaction(f func(*sql.Tx) error, concurrencySafety bool, quiet bool) error { - tx, errBegin := dbw.SqlBegin(concurrencySafety, quiet) - if errBegin != nil { - return errBegin - } - - errTx := f(tx) - if errTx != nil { - dbw.SqlRollback(tx, quiet) - return errTx - } - - return dbw.SqlCommit(tx, quiet) -} - -// SqlTransaction executes the given function inside a transaction. -func (dbw *DBWrapper) SqlTransactionQuiet(concurrencySafety bool, retryOnConnectionFailure bool, f func(*sql.Tx) error) error { - for { - if !dbw.IsConnected() { - dbw.WaitForConnection() - continue - } - - errTx := dbw.sqlTryTransaction(f, concurrencySafety, true) - if errTx != nil { - //TODO: Do this only for concurrencySafety = true, once we figure out the serialization errors. - if isSerializationFailure(errTx) { - continue - } - - if !dbw.checkConnection(false) { - if retryOnConnectionFailure { - continue - } else { - return MysqlConnectionError{"Transaction failed duo to a connection error"} - } - } - - // We still log errors - log.WithFields(log.Fields{ - "context": "sql", - "error": errTx, - }).Warn("SQL error occurred") - } - return errTx + return res, err } } @@ -353,15 +272,90 @@ func (dbw *DBWrapper) SqlRollback(tx DbTransaction, quiet bool) error { } } -// Wrapper around Db.SqlQuery() for auto-logging -func (dbw *DBWrapper) SqlFetchAll(db DbClientOrTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { +// Wrapper around sql.Exec() for auto-logging +func (dbw *DBWrapper) SqlExec(opDescription string, sql string, args ...interface{}) (sql.Result, error) { + return dbw.sqlExecInternal(dbw.Db, opDescription, sql, false, args...) +} + +// No logging, no benchmarking +func (dbw *DBWrapper) SqlExecQuiet(opDescription string, sql string, args ...interface{}) (sql.Result, error) { + return dbw.sqlExecInternal(dbw.Db, opDescription, sql, true, args...) +} + +// Wrapper around tx.Exec() for auto-logging +func (dbw *DBWrapper) SqlExecTx(tx DbTransaction, opDescription string, sql string, args ...interface{}) (sql.Result, error) { + return dbw.sqlExecInternal(tx, opDescription, sql, false, args...) +} + +// No logging, no benchmarking +func (dbw *DBWrapper) SqlExecTxQuiet(tx DbTransaction, opDescription string, sql string, args ...interface{}) (sql.Result, error) { + return dbw.sqlExecInternal(tx, opDescription, sql, true, args...) +} + +func (dbw *DBWrapper) SqlFetchAll(queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + return dbw.sqlFetchAllInternal(dbw.Db, queryDescription, query, false, args...) +} + +func (dbw *DBWrapper) SqlFetchAllQuiet(queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + return dbw.sqlFetchAllInternal(dbw.Db, queryDescription, query, true, args...) +} + +func (dbw *DBWrapper) SqlFetchAllTx(tx DbTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + return dbw.sqlFetchAllInternal(tx, queryDescription, query, false, args...) +} + +func (dbw *DBWrapper) SqlFetchAllTxQuiet(tx DbTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { + return dbw.sqlFetchAllInternal(tx, queryDescription, query, true, args...) +} + +// Wrapper around sql.Exec() for auto-logging +func (dbw *DBWrapper) sqlExecInternal(db DbClientOrTransaction, opDescription string, sql string, quiet bool, args ...interface{}) (sql.Result, error) { for { if !dbw.IsConnected() { dbw.WaitForConnection() continue } - res, err := sqlTryFetchAll(db, queryDescription, query, args...) + var benchmarc *icingadb_utils.Benchmark + if !quiet { + benchmarc = icingadb_utils.NewBenchmark() + } + res, err := db.Exec(sql, args...) + if !quiet { + benchmarc.Stop() + } + + if !quiet { + //DbIoSeconds.WithLabelValues("mysql", opDescription).Observe(benchmarc.Seconds()) + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + "affected_rows": prettyPrintedRowsAffected{res}, + "args": prettyPrintedArgs{args}, + "query": prettyPrintedSql{sql}, + }).Debug("Finished Exec") + } + + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + return res, err + } +} + +// Wrapper around Db.SqlQuery() for auto-logging +func (dbw *DBWrapper) sqlFetchAllInternal(db DbClientOrTransaction, queryDescription string, query string, quiet bool, args ...interface{}) ([][]interface{}, error) { + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + res, err := sqlTryFetchAll(db, queryDescription, query, quiet, args...) if err != nil { if _, isDb := db.(*sql.DB); isDb { @@ -375,23 +369,29 @@ func (dbw *DBWrapper) SqlFetchAll(db DbClientOrTransaction, queryDescription str } } -func sqlTryFetchAll(db DbClientOrTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { - benchmarc := icingadb_utils.NewBenchmark() +func sqlTryFetchAll(db DbClientOrTransaction, queryDescription string, query string, quiet bool, args ...interface{}) ([][]interface{}, error) { + var benchmarc *icingadb_utils.Benchmark + if !quiet { + benchmarc = icingadb_utils.NewBenchmark() + } rows, errQuery := db.Query(query, args...) - benchmarc.Stop() - - //DbIoSeconds.WithLabelValues("mysql", queryDescription).Observe(benchmarc.Seconds()) + if !quiet { + benchmarc.Stop() + } rowsCount := 0 defer func() { - log.WithFields(log.Fields{ - "context": "sql", - "benchmark": benchmarc, - "query": prettyPrintedSql{query}, - "args": prettyPrintedArgs{args}, - "affected_Rows": rowsCount, - }).Debug("Finished FetchAll") + if !quiet { + //DbIoSeconds.WithLabelValues("mysql", queryDescription).Observe(benchmarc.Seconds()) + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + "query": prettyPrintedSql{query}, + "args": prettyPrintedArgs{args}, + "affected_Rows": rowsCount, + }).Debug("Finished FetchAll") + } }() if errQuery != nil { @@ -454,214 +454,74 @@ func sqlTryFetchAll(db DbClientOrTransaction, queryDescription string, query str return res, nil } -// No logging, no benchmarking -func (dbw *DBWrapper) SqlFetchAllQuiet(db DbClientOrTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { +// sqlTransaction executes the given function inside a transaction. +func (dbw DBWrapper) sqlTransactionInternal(concurrencySafety bool, retryOnConnectionFailure bool, quiet bool, f func(*sql.Tx) error) error { for { if !dbw.IsConnected() { dbw.WaitForConnection() continue } - res, err := sqlTryFetchAllQuiet(db, queryDescription, query, args...) + var benchmarc *icingadb_utils.Benchmark + if !quiet { + benchmarc = icingadb_utils.NewBenchmark() + } + errTx := dbw.sqlTryTransaction(f, concurrencySafety, false) + if !quiet { + benchmarc.Stop() + } - if err != nil { - if _, isDb := db.(*sql.DB); isDb { - if !dbw.checkConnection(false) { + //DbIoSeconds.WithLabelValues("mysql", "transaction").Observe(benchmarc.Seconds()) + + if !quiet { + log.WithFields(log.Fields{ + "context": "sql", + "benchmark": benchmarc, + }).Debug("Executed transaction") + } + + if errTx != nil { + //TODO: Do this only for concurrencySafety = true, once we figure out the serialization errors. + if isSerializationFailure(errTx) { + if !quiet { + log.WithFields(log.Fields{ + "context": "sql", + "error": errTx, + }).Debug("Repeating transaction") + } + continue + } + + if !dbw.checkConnection(false) { + if retryOnConnectionFailure { continue + } else { + return MysqlConnectionError{"Transaction failed duo to a connection error"} } } + + log.WithFields(log.Fields{ + "context": "sql", + "error": errTx, + }).Warn("SQL error occurred") } - return res, err + return errTx } } -func sqlTryFetchAllQuiet(db DbClientOrTransaction, queryDescription string, query string, args ...interface{}) ([][]interface{}, error) { - rows, errQuery := db.Query(query, args...) - - if errQuery != nil { - return [][]interface{}{}, errQuery +// Executes the given function inside a transaction +func (dbw *DBWrapper) sqlTryTransaction(f func(*sql.Tx) error, concurrencySafety bool, quiet bool) error { + tx, errBegin := dbw.SqlBegin(concurrencySafety, quiet) + if errBegin != nil { + return errBegin } - defer rows.Close() - - columnTypes, errCT := rows.ColumnTypes() - if errCT != nil { - return [][]interface{}{}, errCT + errTx := f(tx) + if errTx != nil { + dbw.SqlRollback(tx, quiet) + return errTx } - colsPerRow := len(columnTypes) - buf := list.New() - bridges := make([]dbTypeBridge, colsPerRow) - scanDest := make([]interface{}, colsPerRow) - - for i, columnType := range columnTypes { - typ := columnType.DatabaseTypeName() - factory, hasFactory := dbTypeBridgeFactories[typ] - if hasFactory { - bridges[i] = factory() - } else { - bridges[i] = &dbBrokenBridge{typ: typ} - } - - scanDest[i] = bridges[i] - } - - for { - if rows.Next() { - if errScan := rows.Scan(scanDest...); errScan != nil { - return [][]interface{}{}, errScan - } - - row := make([]interface{}, colsPerRow) - - for i, bridge := range bridges { - row[i] = bridge.Result() - } - - buf.PushBack(row) - } else if errNx := rows.Err(); errNx == nil { - break - } else { - return nil, errNx - } - } - - res := make([][]interface{}, buf.Len()) - - for current, i := buf.Front(), 0; current != nil; current = current.Next() { - res[i] = current.Value.([]interface{}) - i++ - } - - return res, nil -} - -// Wrapper around tx.Exec() for auto-logging -func (dbw *DBWrapper) SqlExecTx(tx DbTransaction, opDescription string, sql string, args ...interface{}) (sql.Result, error) { - for { - if !dbw.IsConnected() { - dbw.WaitForConnection() - continue - } - - benchmarc := icingadb_utils.NewBenchmark() - res, err := tx.Exec(sql, args...) - benchmarc.Stop() - - //DbIoSeconds.WithLabelValues("mysql", opDescription).Observe(benchmarc.Seconds()) - - log.WithFields(log.Fields{ - "context": "sql", - "benchmark": benchmarc, - "affected_rows": prettyPrintedRowsAffected{res}, - "args": prettyPrintedArgs{args}, - "query": prettyPrintedSql{sql}, - }).Debug("Finished Exec") - - - if err != nil { - if !dbw.checkConnection(false) { - continue - } - } - - return res, err - } -} - -// No logging, no benchmarking -func (dbw *DBWrapper) SqlExecTxQuiet(tx DbTransaction, opDescription string, sql string, args ...interface{}) (sql.Result, error) { - for { - if !dbw.IsConnected() { - dbw.WaitForConnection() - continue - } - - res, err := tx.Exec(sql, args...) - - if err != nil { - if !dbw.checkConnection(false) { - continue - } - } - - return res, err - } -} - -// Wrapper around sql.Exec() for auto-logging -func (dbw *DBWrapper) SqlExec(opDescription string, sql string, args ...interface{}) (sql.Result, error) { - for { - if !dbw.IsConnected() { - dbw.WaitForConnection() - continue - } - - benchmarc := icingadb_utils.NewBenchmark() - res, err := dbw.Db.Exec(sql, args...) - benchmarc.Stop() - - //DbIoSeconds.WithLabelValues("mysql", opDescription).Observe(benchmarc.Seconds()) - - log.WithFields(log.Fields{ - "context": "sql", - "benchmark": benchmarc, - "affected_rows": prettyPrintedRowsAffected{res}, - "args": prettyPrintedArgs{args}, - "query": prettyPrintedSql{sql}, - }).Debug("Finished Exec") - - - if err != nil { - if !dbw.checkConnection(false) { - continue - } - } - - return res, err - } -} - -// No logging, no benchmarking -func (dbw *DBWrapper) SqlExecQuiet(opDescription string, sql string, args ...interface{}) (sql.Result, error) { - for { - if !dbw.IsConnected() { - dbw.WaitForConnection() - continue - } - - res, err := dbw.Db.Exec(sql, args...) - - if err != nil { - if !dbw.checkConnection(false) { - continue - } - } - - return res, err - } -} - -func IsRetryableError(err error) bool { - if strings.Contains(err.Error(), "Deadlock found when trying to get lock") { - return true - } - return false -} - -func (dbw *DBWrapper) WithRetry(f func() (sql.Result, error)) (sql.Result, error) { - for { - res, err := f() - - if err != nil { - if IsRetryableError(err) { - continue - } else { - return nil, err - } - } - - return res, err - } + return dbw.SqlCommit(tx, quiet) } \ No newline at end of file diff --git a/mysql_utils.go b/mysql_utils.go index 2638dd3e..e991b247 100644 --- a/mysql_utils.go +++ b/mysql_utils.go @@ -364,3 +364,10 @@ func MakePlaceholderList(x int) string { return string(runes) } + +func isRetryableError(err error) bool { + if strings.Contains(err.Error(), "Deadlock found when trying to get lock") { + return true + } + return false +} \ No newline at end of file From ed6224e4420bcd01aa0c5aefcf29d93062be4e02 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 11:15:18 +0100 Subject: [PATCH 21/69] Fix transaction --- mysql.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql.go b/mysql.go index a3df3eaf..3ac5b694 100644 --- a/mysql.go +++ b/mysql.go @@ -163,7 +163,7 @@ func (dbw *DBWrapper) SqlQuery(query string, args ...interface{}) (*sql.Rows, er } // Wrapper around Db.BeginTx() for auto-logging -func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (*sql.Tx, error) { +func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (DbTransaction, error) { var isoLvl sql.IsolationLevel if concurrencySafety { isoLvl = sql.LevelSerializable @@ -455,7 +455,7 @@ func sqlTryFetchAll(db DbClientOrTransaction, queryDescription string, query str } // sqlTransaction executes the given function inside a transaction. -func (dbw DBWrapper) sqlTransactionInternal(concurrencySafety bool, retryOnConnectionFailure bool, quiet bool, f func(*sql.Tx) error) error { +func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFailure bool, quiet bool, f func(DbTransaction) error) error { for { if !dbw.IsConnected() { dbw.WaitForConnection() @@ -511,7 +511,7 @@ func (dbw DBWrapper) sqlTransactionInternal(concurrencySafety bool, retryOnConne } // Executes the given function inside a transaction -func (dbw *DBWrapper) sqlTryTransaction(f func(*sql.Tx) error, concurrencySafety bool, quiet bool) error { +func (dbw *DBWrapper) sqlTryTransaction(f func(transaction DbTransaction) error, concurrencySafety bool, quiet bool) error { tx, errBegin := dbw.SqlBegin(concurrencySafety, quiet) if errBegin != nil { return errBegin From d8e6e8955de024702e6b0751399ee1cfc1b83289 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 11:15:43 +0100 Subject: [PATCH 22/69] Add more MySQL tests --- mysql_test.go | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/mysql_test.go b/mysql_test.go index 4b355d43..d1998edf 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -15,6 +15,7 @@ import ( type SqlResultMock struct { sql.Result } + type TransactionMock struct { mock.Mock } @@ -95,6 +96,64 @@ func TestRDBWrapper_CheckConnection(t *testing.T) { assert.Equal(t, 11, dbw.ConnectionLostCounter) } +func TestDBWrapper_SqlCommit(t *testing.T) { + mockDb := new(DbMock) + dbw := NewTestDBW(mockDb) + mockTx := new(TransactionMock) + + mockTx.On("Commit").Return(errors.New("whoops")).Once() + mockTx.On("Commit").Return( nil).Once() + mockDb.On("Ping").Return(errors.New("whoops")).Once() + + var err error + done := make(chan bool) + + dbw.CompareAndSetConnected(true) + go func() { + err = dbw.SqlCommit(mockTx, false) + done <- true + }() + + time.Sleep(time.Millisecond * 100) + + dbw.CompareAndSetConnected(true) + dbw.ConnectionUpCondition.Broadcast() + + <- done + + assert.Nil(t, err) + mockTx.AssertExpectations(t) + mockDb.AssertExpectations(t) +} + +func TestDBWrapper_SqlBegin(t *testing.T) { + mockDb := new(DbMock) + dbw := NewTestDBW(mockDb) + + mockDb.On("BeginTx", context.Background(), &sql.TxOptions{Isolation: sql.LevelReadCommitted}).Return(&sql.Tx{}, errors.New("whoops")).Once() + mockDb.On("BeginTx", context.Background(), &sql.TxOptions{Isolation: sql.LevelReadCommitted}).Return(&sql.Tx{}, nil).Once() + mockDb.On("Ping").Return(errors.New("whoops")).Once() + + var err error + done := make(chan bool) + + dbw.CompareAndSetConnected(true) + go func() { + _, err = dbw.SqlBegin(false, false) + done <- true + }() + + time.Sleep(time.Millisecond * 100) + + dbw.CompareAndSetConnected(true) + dbw.ConnectionUpCondition.Broadcast() + + <- done + + assert.Nil(t, err) + mockDb.AssertExpectations(t) +} + func TestDBWrapper_WithRetry(t *testing.T) { mockDb := new(DbMock) dbw := NewTestDBW(mockDb) @@ -113,6 +172,12 @@ func TestDBWrapper_WithRetry(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 2, tries) + + _, err = dbw.WithRetry(func() (result sql.Result, e error) { + return nil, errors.New("something went wrong") + }) + + assert.Error(t, err) } func TestDBWrapper_SqlQuery(t *testing.T) { From 137f77eef60e949c86ed5c67d000f4c9ee413c12 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 13:54:23 +0100 Subject: [PATCH 23/69] Fix race condition --- mysql.go | 25 +++++++++++++------------ mysql_test.go | 23 ++++++++++++----------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/mysql.go b/mysql.go index 3ac5b694..dd78971b 100644 --- a/mysql.go +++ b/mysql.go @@ -36,7 +36,7 @@ func NewDBWrapper(dbType string, dbDsn string) (*DBWrapper, error) { return nil, err } - dbw := DBWrapper{Db: db, ConnectedAtomic: new(uint32)} + dbw := DBWrapper{Db: db, ConnectedAtomic: new(uint32), ConnectionLostCounterAtomic: new(uint32)} dbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) err = dbw.Db.Ping() @@ -56,10 +56,10 @@ func NewDBWrapper(dbType string, dbDsn string) (*DBWrapper, error) { // Database wrapper including helper functions type DBWrapper struct { - Db DbClient - ConnectedAtomic *uint32 //uint32 to be able to use atomic operations - ConnectionUpCondition *sync.Cond - ConnectionLostCounter int + Db DbClient + ConnectedAtomic *uint32 //uint32 to be able to use atomic operations + ConnectionUpCondition *sync.Cond + ConnectionLostCounterAtomic *uint32 //uint32 to be able to use atomic operations } func (dbw *DBWrapper) IsConnected() bool { @@ -76,13 +76,14 @@ func (dbw *DBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { func (dbw *DBWrapper) getConnectionCheckInterval() time.Duration { if !dbw.IsConnected() { - if dbw.ConnectionLostCounter < 4 { + v := atomic.LoadUint32(dbw.ConnectionLostCounterAtomic) + if v < 4 { return 5 * time.Second - } else if dbw.ConnectionLostCounter < 8 { + } else if v < 8 { return 10 * time.Second - } else if dbw.ConnectionLostCounter < 11 { + } else if v < 11 { return 30 * time.Second - } else if dbw.ConnectionLostCounter < 14 { + } else if v < 14 { return 60 * time.Second } else { log.Fatal("Could not connect to SQL for over 5 minutes. Shutting down...") @@ -101,7 +102,7 @@ func (dbw *DBWrapper) checkConnection(isTicker bool) bool { "error": err, }).Error("SQL connection lost. Trying to reconnect") } else if isTicker { - dbw.ConnectionLostCounter++ + atomic.AddUint32(dbw.ConnectionLostCounterAtomic, 1) log.WithFields(log.Fields{ "context": "sql", @@ -113,7 +114,7 @@ func (dbw *DBWrapper) checkConnection(isTicker bool) bool { } else { if dbw.CompareAndSetConnected(true) { log.Info("SQL connection established") - dbw.ConnectionLostCounter = 0 + atomic.StoreUint32(dbw.ConnectionLostCounterAtomic, 0) dbw.ConnectionUpCondition.Broadcast() } @@ -178,7 +179,7 @@ func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (DbTransactio } var err error - var tx *sql.Tx + var tx DbTransaction if quiet { tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) } else { diff --git a/mysql_test.go b/mysql_test.go index d1998edf..8a9a452a 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "sync" + "sync/atomic" "testing" "time" ) @@ -65,7 +66,7 @@ func (m *DbMock) Exec(query string, args ...interface{}) (sql.Result, error) { } func NewTestDBW(db DbClient) DBWrapper { - dbw := DBWrapper{Db: db, ConnectedAtomic: new(uint32)} + dbw := DBWrapper{Db: db, ConnectedAtomic: new(uint32), ConnectionLostCounterAtomic: new(uint32)} dbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) return dbw } @@ -80,20 +81,20 @@ func TestRDBWrapper_CheckConnection(t *testing.T) { mockDb := new(DbMock) dbw := NewTestDBW(mockDb) - dbw.ConnectionLostCounter = 180239812 + atomic.StoreUint32(dbw.ConnectionLostCounterAtomic, 512312312) mockDb.On("Ping").Return(nil).Once() assert.True(t, dbw.checkConnection(false), "DBWrapper should be connected") - assert.Equal(t, 0, dbw.ConnectionLostCounter) + assert.Equal(t, uint32(0), atomic.LoadUint32(dbw.ConnectionLostCounterAtomic)) - dbw.ConnectionLostCounter = 0 + atomic.StoreUint32(dbw.ConnectionLostCounterAtomic, 0) mockDb.On("Ping").Return(mysql.ErrInvalidConn).Once() assert.False(t, dbw.checkConnection(false), "DBWrapper should not be connected") - assert.Equal(t, 0, dbw.ConnectionLostCounter) + assert.Equal(t, uint32(0), atomic.LoadUint32(dbw.ConnectionLostCounterAtomic)) - dbw.ConnectionLostCounter = 10 + atomic.StoreUint32(dbw.ConnectionLostCounterAtomic, 10) mockDb.On("Ping").Return(mysql.ErrInvalidConn).Once() assert.False(t, dbw.checkConnection(true), "DBWrapper should not be connected") - assert.Equal(t, 11, dbw.ConnectionLostCounter) + assert.Equal(t, uint32(11), atomic.LoadUint32(dbw.ConnectionLostCounterAtomic)) } func TestDBWrapper_SqlCommit(t *testing.T) { @@ -333,22 +334,22 @@ func TestGetConnectionCheckInterval(t *testing.T) { //Should return 5s, if not connected and counter < 4 dbw.CompareAndSetConnected(false) - dbw.ConnectionLostCounter = 0 + atomic.StoreUint32(dbw.ConnectionLostCounterAtomic, 0) assert.Equal(t, 5*time.Second, dbw.getConnectionCheckInterval()) //Should return 10s, if not connected and 4 <= counter < 8 dbw.CompareAndSetConnected(false) - dbw.ConnectionLostCounter = 4 + atomic.StoreUint32(dbw.ConnectionLostCounterAtomic, 4) assert.Equal(t, 10*time.Second, dbw.getConnectionCheckInterval()) //Should return 30s, if not connected and 8 <= counter < 11 dbw.CompareAndSetConnected(false) - dbw.ConnectionLostCounter = 8 + atomic.StoreUint32(dbw.ConnectionLostCounterAtomic, 8) assert.Equal(t, 30*time.Second, dbw.getConnectionCheckInterval()) //Should return 60s, if not connected and 11 <= counter < 14 dbw.CompareAndSetConnected(false) - dbw.ConnectionLostCounter = 11 + atomic.StoreUint32(dbw.ConnectionLostCounterAtomic, 11) assert.Equal(t, 60*time.Second, dbw.getConnectionCheckInterval()) //dbw.ConnectionLostCounter = 14 From 92e81ade151da62c1f89daee5bac47fba1a17c4a Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 14:27:06 +0100 Subject: [PATCH 24/69] Add prometheus - WIP --- mysql.go | 15 +++++++++------ prometheus.go | 39 +++++++++++++++++++++++++++++++++++++++ redis.go | 4 ++-- 3 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 prometheus.go diff --git a/mysql.go b/mysql.go index dd78971b..d8f11733 100644 --- a/mysql.go +++ b/mysql.go @@ -152,6 +152,7 @@ func (dbw *DBWrapper) SqlQuery(query string, args ...interface{}) (*sql.Rows, er } res, err := dbw.Db.Query(query, args...) + DbOperationsQuery.Inc() if err != nil { if !dbw.checkConnection(false) { @@ -187,7 +188,7 @@ func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (DbTransactio tx, err = dbw.Db.BeginTx(context.Background(), &sql.TxOptions{Isolation: isoLvl}) benchmarc.Stop() - //DbIoSeconds.WithLabelValues("mysql", "begin").Observe(benchmarc.Seconds()) + DbIoSeconds.WithLabelValues("mysql", "begin").Observe(benchmarc.Seconds()) log.WithFields(log.Fields{ "context": "sql", @@ -221,7 +222,7 @@ func (dbw *DBWrapper) SqlCommit(tx DbTransaction, quiet bool) error { err = tx.Commit() benchmarc.Stop() - //DbIoSeconds.WithLabelValues("mysql", "commit").Observe(benchmarc.Seconds()) + DbIoSeconds.WithLabelValues("mysql", "commit").Observe(benchmarc.Seconds()) log.WithFields(log.Fields{ "context": "sql", @@ -253,7 +254,7 @@ func (dbw *DBWrapper) SqlRollback(tx DbTransaction, quiet bool) error { err = tx.Rollback() benchmarc.Stop() - //DbIoSeconds.WithLabelValues("mysql", "rollback").Observe(benchmarc.Seconds()) + DbIoSeconds.WithLabelValues("mysql", "rollback").Observe(benchmarc.Seconds()) log.WithFields(log.Fields{ "context": "sql", @@ -322,12 +323,13 @@ func (dbw *DBWrapper) sqlExecInternal(db DbClientOrTransaction, opDescription st benchmarc = icingadb_utils.NewBenchmark() } res, err := db.Exec(sql, args...) + DbOperationsExec.Inc() if !quiet { benchmarc.Stop() } if !quiet { - //DbIoSeconds.WithLabelValues("mysql", opDescription).Observe(benchmarc.Seconds()) + DbIoSeconds.WithLabelValues("mysql", opDescription).Observe(benchmarc.Seconds()) log.WithFields(log.Fields{ "context": "sql", "benchmark": benchmarc, @@ -376,6 +378,7 @@ func sqlTryFetchAll(db DbClientOrTransaction, queryDescription string, query str benchmarc = icingadb_utils.NewBenchmark() } rows, errQuery := db.Query(query, args...) + DbOperationsQuery.Inc() if !quiet { benchmarc.Stop() } @@ -384,7 +387,7 @@ func sqlTryFetchAll(db DbClientOrTransaction, queryDescription string, query str defer func() { if !quiet { - //DbIoSeconds.WithLabelValues("mysql", queryDescription).Observe(benchmarc.Seconds()) + DbIoSeconds.WithLabelValues("mysql", queryDescription).Observe(benchmarc.Seconds()) log.WithFields(log.Fields{ "context": "sql", "benchmark": benchmarc, @@ -472,7 +475,7 @@ func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFai benchmarc.Stop() } - //DbIoSeconds.WithLabelValues("mysql", "transaction").Observe(benchmarc.Seconds()) + DbIoSeconds.WithLabelValues("mysql", "transaction").Observe(benchmarc.Seconds()) if !quiet { log.WithFields(log.Fields{ diff --git a/prometheus.go b/prometheus.go new file mode 100644 index 00000000..f977c88a --- /dev/null +++ b/prometheus.go @@ -0,0 +1,39 @@ +package icingadb_connection + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" + "net/http" +) + +var DbIoSeconds = promauto.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "db_io_seconds", + Help: "Database I/O (s)", + }, + []string{"backend_type", "operation"}, +) + +var DbOperationsTotal = promauto.NewCounter(prometheus.CounterOpts{ + Name: "db_operations_total", + Help: "Database operations since startup", +}) + +var DbOperationsQuery = promauto.NewCounter(prometheus.CounterOpts{ + Name: "db_operations_query", + Help: "Database query operations since startup", +}) + +var DbOperationsExec = promauto.NewCounter(prometheus.CounterOpts{ + Name: "db_operations_exec", + Help: "Database exec operations since startup", +}) + +//TODO: Move this to main package of IcingaDB +func Httpd(addr string, chErr chan error) { + http.Handle("/metrics", promhttp.Handler()) + log.Infof("Serving debug info at http://%s/metrics", addr) + chErr <- http.ListenAndServe(addr, nil) +} diff --git a/redis.go b/redis.go index d28a371d..e6dd48cf 100644 --- a/redis.go +++ b/redis.go @@ -230,7 +230,7 @@ func (rdbw *RDBWrapper) HGetAll(key string) (map[string]string, error) { benchmarc.Stop() - //DbIoSeconds.WithLabelValues("redis", "hgetall").Observe(benchmarc.Seconds()) + DbIoSeconds.WithLabelValues("redis", "hgetall").Observe(benchmarc.Seconds()) log.WithFields(log.Fields{ "context": "redis", @@ -261,7 +261,7 @@ func (rdbw *RDBWrapper) TxPipelined(fn func(pipeliner redis.Pipeliner) error) ([ benchmarc.Stop() - //DbIoSeconds.WithLabelValues("redis", "multi").Observe(benchmarc.Seconds()) + DbIoSeconds.WithLabelValues("redis", "multi").Observe(benchmarc.Seconds()) log.WithFields(log.Fields{ "context": "redis", From 39e136f1a84c7da9160ec60272ce60a12eea59fa Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 14:28:11 +0100 Subject: [PATCH 25/69] Add SqlTransaction() Test --- mysql_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mysql_test.go b/mysql_test.go index 8a9a452a..f0a3568b 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -155,6 +155,23 @@ func TestDBWrapper_SqlBegin(t *testing.T) { mockDb.AssertExpectations(t) } +func TestDBWrapper_SqlTransaction(t *testing.T) { + dbw, err := NewDBWrapper("mysql", "icingadb:icingadb@tcp(127.0.0.1:3306)/icingadb") + assert.NoError(t, err, "Is the MySQL server running?") + + err = dbw.SqlTransaction(false, true, false, func(tx DbTransaction) error { + return nil + }) + + assert.NoError(t, err) + + err = dbw.SqlTransaction(false, true, false, func(tx DbTransaction) error { + return errors.New("whoops") + }) + + assert.Error(t, err) +} + func TestDBWrapper_WithRetry(t *testing.T) { mockDb := new(DbMock) dbw := NewTestDBW(mockDb) From a776d64f9c10df43f9be5b4f346f6c41e09918c0 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 14:28:36 +0100 Subject: [PATCH 26/69] Use Error/NoError instead of NotNil/Nil --- mysql_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mysql_test.go b/mysql_test.go index f0a3568b..84b7cbc8 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -73,7 +73,7 @@ func NewTestDBW(db DbClient) DBWrapper { func TestNewDBWrapper(t *testing.T) { _, err := NewDBWrapper("mysql", "asdasd") - assert.NotNil(t, err) + assert.Error(t, err) //TODO: Add more tests here } @@ -122,7 +122,7 @@ func TestDBWrapper_SqlCommit(t *testing.T) { <- done - assert.Nil(t, err) + assert.NoError(t, err) mockTx.AssertExpectations(t) mockDb.AssertExpectations(t) } @@ -151,7 +151,7 @@ func TestDBWrapper_SqlBegin(t *testing.T) { <- done - assert.Nil(t, err) + assert.NoError(t, err) mockDb.AssertExpectations(t) } @@ -188,7 +188,7 @@ func TestDBWrapper_WithRetry(t *testing.T) { } }) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 2, tries) _, err = dbw.WithRetry(func() (result sql.Result, e error) { @@ -222,7 +222,7 @@ func TestDBWrapper_SqlQuery(t *testing.T) { <- done - assert.Nil(t, err) + assert.NoError(t, err) mockDb.AssertExpectations(t) } @@ -250,7 +250,7 @@ func TestDBWrapper_SqlExec(t *testing.T) { <- done - assert.Nil(t, err) + assert.NoError(t, err) mockDb.AssertExpectations(t) } @@ -278,7 +278,7 @@ func TestDBWrapper_SqlExecQuiet(t *testing.T) { <- done - assert.Nil(t, err) + assert.NoError(t, err) mockDb.AssertExpectations(t) } @@ -307,7 +307,7 @@ func TestDBWrapper_SqlExecTx(t *testing.T) { <- done - assert.Nil(t, err) + assert.NoError(t, err) mockTx.AssertExpectations(t) mockDb.AssertExpectations(t) } @@ -337,7 +337,7 @@ func TestDBWrapper_SqlExecTxQuiet(t *testing.T) { <- done - assert.Nil(t, err) + assert.NoError(t, err) mockTx.AssertExpectations(t) mockDb.AssertExpectations(t) } From 6299908ed3e6b8096541d59c8a6412fb49dd862c Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 14:29:33 +0100 Subject: [PATCH 27/69] Add coverage.sh --- .gitignore | 2 ++ coverage.sh | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 .gitignore create mode 100755 coverage.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..3d800336 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +coverage\.html diff --git a/coverage.sh b/coverage.sh new file mode 100755 index 00000000..4b35f56d --- /dev/null +++ b/coverage.sh @@ -0,0 +1,3 @@ +go test -race -cover -coverprofile=c.out +go tool cover -html=c.out -o coverage.html +rm c.out From 789e5132174772a61c47592a0233e05a38c723ed Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 14:42:35 +0100 Subject: [PATCH 28/69] Add coverage report --- README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..e0fc2a50 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +IcingaDB Connection Library + +[![pipeline status](https://git.icinga.com/icingadb/icingadb-connection-lib/badges/master/pipeline.svg)](https://git.icinga.com/icingadb/icingadb-connection-lib/commits/master) +[![coverage report](https://git.icinga.com/icingadb/icingadb-connection-lib/badges/master/coverage.svg)](https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/artifacts/master/browse?job=coverage) \ No newline at end of file From a245049ffe32bde62ff2e9760e14d4ada62e1f9e Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 15:05:05 +0100 Subject: [PATCH 29/69] Add temporary mysql-server to test/coverage-containers --- test_db.sql | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 test_db.sql diff --git a/test_db.sql b/test_db.sql new file mode 100644 index 00000000..b639f785 --- /dev/null +++ b/test_db.sql @@ -0,0 +1,3 @@ +CREATE database icingadb; +CREATE USER 'icingadb'@'127.0.0.1' IDENTIFIED BY 'icingadb'; +GRANT ALL PRIVILEGES ON icingadb.* TO 'icingadb'@'127.0.0.1'; \ No newline at end of file From 93198414a0a032c2a494b0c25db0cb543069b251 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 26 Feb 2019 15:10:37 +0100 Subject: [PATCH 30/69] Fix coverage link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e0fc2a50..57eedb83 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ IcingaDB Connection Library [![pipeline status](https://git.icinga.com/icingadb/icingadb-connection-lib/badges/master/pipeline.svg)](https://git.icinga.com/icingadb/icingadb-connection-lib/commits/master) -[![coverage report](https://git.icinga.com/icingadb/icingadb-connection-lib/badges/master/coverage.svg)](https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/artifacts/master/browse?job=coverage) \ No newline at end of file +[![coverage report](https://git.icinga.com/icingadb/icingadb-connection-lib/badges/master/coverage.svg)](https://git.icinga.com/icingadb/icingadb-connection-lib/-/jobs/artifacts/master/raw/coverage.html?job=coverage) \ No newline at end of file From 842cf3cec9a79ab696fc98726040e3877b7cdbf4 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 11:37:03 +0100 Subject: [PATCH 31/69] Use project global credentials --- mysql_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql_test.go b/mysql_test.go index 84b7cbc8..92c2e0c7 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -156,7 +156,7 @@ func TestDBWrapper_SqlBegin(t *testing.T) { } func TestDBWrapper_SqlTransaction(t *testing.T) { - dbw, err := NewDBWrapper("mysql", "icingadb:icingadb@tcp(127.0.0.1:3306)/icingadb") + dbw, err := NewDBWrapper("mysql", "module-dev:icinga0815!@tcp(127.0.0.1:3306)/icingadb") assert.NoError(t, err, "Is the MySQL server running?") err = dbw.SqlTransaction(false, true, false, func(tx DbTransaction) error { From cb018cea9d4f04bb02298b6c8c65633f3626472b Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 11:39:49 +0100 Subject: [PATCH 32/69] Fix Gitlab-CI --- test_db.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_db.sql b/test_db.sql index b639f785..69cbd78d 100644 --- a/test_db.sql +++ b/test_db.sql @@ -1,3 +1,3 @@ CREATE database icingadb; -CREATE USER 'icingadb'@'127.0.0.1' IDENTIFIED BY 'icingadb'; -GRANT ALL PRIVILEGES ON icingadb.* TO 'icingadb'@'127.0.0.1'; \ No newline at end of file +CREATE USER 'module-dev'@'127.0.0.1' IDENTIFIED BY 'icinga0815!'; +GRANT ALL PRIVILEGES ON icingadb.* TO 'module-dev'@'127.0.0.1'; \ No newline at end of file From dce9c0fa4f80245a5564cb68bfb58b32e47755bc Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 12:18:48 +0100 Subject: [PATCH 33/69] Redis: Check for connection on NewRDBWrapper() --- redis.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/redis.go b/redis.go index e6dd48cf..816456b6 100644 --- a/redis.go +++ b/redis.go @@ -82,10 +82,15 @@ func (rdbw *RDBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { } } -func NewRDBWrapper(rdb *redis.Client) *RDBWrapper { +func NewRDBWrapper(rdb *redis.Client) (*RDBWrapper, error) { rdbw := RDBWrapper{Rdb: rdb, ConnectedAtomic: new(uint32)} rdbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) + res := rdbw.Rdb.Ping() + if res.Err() != nil { + return nil, res.Err() + } + go func() { for { rdbw.CheckConnection(true) @@ -93,7 +98,7 @@ func NewRDBWrapper(rdb *redis.Client) *RDBWrapper { } }() - return &rdbw + return &rdbw, nil } func (rdbw *RDBWrapper) getConnectionCheckInterval() time.Duration { From 615d255f7efcab0e3f66fd993aa710c41a843a65 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 13:29:02 +0100 Subject: [PATCH 34/69] NewRDBWrapper() should create Redis-Client --- redis.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/redis.go b/redis.go index 816456b6..1f8f57a7 100644 --- a/redis.go +++ b/redis.go @@ -82,13 +82,19 @@ func (rdbw *RDBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { } } -func NewRDBWrapper(rdb *redis.Client) (*RDBWrapper, error) { +func NewRDBWrapper(address string) (*RDBWrapper, error) { + rdb := redis.NewClient(&redis.Options{ + Addr: address, + DialTimeout: time.Minute / 2, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + }) + rdbw := RDBWrapper{Rdb: rdb, ConnectedAtomic: new(uint32)} rdbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) - - res := rdbw.Rdb.Ping() - if res.Err() != nil { - return nil, res.Err() + _, err := rdbw.Rdb.Ping().Result() + if err != nil { + return nil, err } go func() { From 995b8e49ee87932bc16c6d8cbf9e4176a58eda03 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 15:41:54 +0100 Subject: [PATCH 35/69] Redis: Use atomic counters to prevent race conditions --- redis.go | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/redis.go b/redis.go index 1f8f57a7..331389a0 100644 --- a/redis.go +++ b/redis.go @@ -62,12 +62,26 @@ var RedisWriter = Icinga2RedisWriter{ }, } +type RedisClient interface { + Ping() *redis.StatusCmd + Publish(channel string, message interface{}) *redis.IntCmd + XRead(a *redis.XReadArgs) *redis.XStreamSliceCmd + XDel(stream string, ids ...string) *redis.IntCmd + HGetAll(key string) *redis.StringStringMapCmd + TxPipelined(fn func(redis.Pipeliner) error) ([]redis.Cmder, error) + Subscribe(channels ...string) *redis.PubSub +} + +type StatusCmd interface { + +} + // Redis wrapper including helper functions type RDBWrapper struct { - Rdb *redis.Client + Rdb RedisClient ConnectedAtomic *uint32 //uint32 to be able to use atomic operations ConnectionUpCondition *sync.Cond - ConnectionLostCounter int + ConnectionLostCounterAtomic *uint32 //uint32 to be able to use atomic operations } func (rdbw *RDBWrapper) IsConnected() bool { @@ -90,8 +104,12 @@ func NewRDBWrapper(address string) (*RDBWrapper, error) { WriteTimeout: time.Minute, }) - rdbw := RDBWrapper{Rdb: rdb, ConnectedAtomic: new(uint32)} - rdbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) + rdbw := RDBWrapper{ + Rdb: rdb, ConnectedAtomic: new(uint32), + ConnectionLostCounterAtomic: new(uint32), + ConnectionUpCondition: sync.NewCond(&sync.Mutex{}), + } + _, err := rdbw.Rdb.Ping().Result() if err != nil { return nil, err @@ -109,13 +127,14 @@ func NewRDBWrapper(address string) (*RDBWrapper, error) { func (rdbw *RDBWrapper) getConnectionCheckInterval() time.Duration { if !rdbw.IsConnected() { - if rdbw.ConnectionLostCounter < 4 { + v := atomic.LoadUint32(rdbw.ConnectionLostCounterAtomic) + if v < 4 { return 5 * time.Second - } else if rdbw.ConnectionLostCounter < 8 { + } else if v < 8 { return 10 * time.Second - } else if rdbw.ConnectionLostCounter < 11 { + } else if v < 11 { return 30 * time.Second - } else if rdbw.ConnectionLostCounter < 14 { + } else if v < 14 { return 60 * time.Second } else { log.Fatal("Could not connect to Redis for over 5 minutes. Shutting down...") @@ -134,7 +153,7 @@ func (rdbw *RDBWrapper) CheckConnection(isTicker bool) bool { "error": err, }).Error("Redis connection lost. Trying to reconnect") } else if isTicker { - rdbw.ConnectionLostCounter++ + atomic.AddUint32(rdbw.ConnectionLostCounterAtomic, 1) log.WithFields(log.Fields{ "context": "redis", @@ -146,7 +165,7 @@ func (rdbw *RDBWrapper) CheckConnection(isTicker bool) bool { } else { if rdbw.CompareAndSetConnected(true) { log.Info("Redis connection established") - rdbw.ConnectionLostCounter = 0 + atomic.StoreUint32(rdbw.ConnectionLostCounterAtomic, 0) rdbw.ConnectionUpCondition.Broadcast() } @@ -259,8 +278,9 @@ func (rdbw *RDBWrapper) TxPipelined(fn func(pipeliner redis.Pipeliner) error) ([ for { if !rdbw.IsConnected() { rdbw.WaitForConnection() - continue + continue } + benchmarc := icingadb_utils.NewBenchmark() c, e := rdbw.Rdb.TxPipelined(fn) From d090d2426124a4950c2cab22977557ecb8726b70 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 15:42:11 +0100 Subject: [PATCH 36/69] Redis: Add first tests --- redis_test.go | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 redis_test.go diff --git a/redis_test.go b/redis_test.go new file mode 100644 index 00000000..2a699b04 --- /dev/null +++ b/redis_test.go @@ -0,0 +1,125 @@ +package icingadb_connection + +import ( + "github.com/go-redis/redis" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "sync" + "sync/atomic" + "testing" + "time" +) + +type RdbMock struct { + mock.Mock +} + +func (m *RdbMock) Ping() *redis.StatusCmd { + args := m.Called() + return args.Get(0).(*redis.StatusCmd) +} + +func (m *RdbMock) Publish(channel string, message interface{}) *redis.IntCmd { + args := m.Called(channel, message) + return args.Get(0).(*redis.IntCmd) +} + +func (m *RdbMock) XRead(a *redis.XReadArgs) *redis.XStreamSliceCmd { + args := m.Called(a) + return args.Get(0).(*redis.XStreamSliceCmd) +} + +func (m *RdbMock) XDel(stream string, ids ...string) *redis.IntCmd { + args := m.Called(stream, ids) + return args.Get(0).(*redis.IntCmd) +} + +func (m *RdbMock) HGetAll(key string) *redis.StringStringMapCmd { + args := m.Called(key) + return args.Get(0).(*redis.StringStringMapCmd) +} + +func (m *RdbMock) TxPipelined(fn func(redis.Pipeliner) error) ([]redis.Cmder, error) { + args := m.Called(fn) + return args.Get(0).([]redis.Cmder), args.Error(1) +} + +func (m *RdbMock) Subscribe(channels ...string) *redis.PubSub { + args := m.Called(channels) + return args.Get(0).(*redis.PubSub) +} + +func NewTestRDBW(rdb RedisClient) RDBWrapper { + dbw := RDBWrapper{Rdb: rdb, ConnectedAtomic: new(uint32), ConnectionLostCounterAtomic: new(uint32)} + dbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) + return dbw +} + +func TestNewRDBWrapper(t *testing.T) { + _, err := NewRDBWrapper("127.0.0.1:6379") + assert.NoError(t, err, "Redis should be connected") + + _, err = NewRDBWrapper("asdasdasdasdasd:5123") + assert.Error(t, err, "Redis should not be connected") + //TODO: Add more tests here +} + +func TestRDBWrapper_GetConnectionCheckInterval(t *testing.T) { + rdbw := NewTestRDBW(nil) + + //Should return 15s, if connected - counter doesn't madder + rdbw.CompareAndSetConnected(true) + assert.Equal(t, 15*time.Second, rdbw.getConnectionCheckInterval()) + + //Should return 5s, if not connected and counter < 4 + rdbw.CompareAndSetConnected(false) + atomic.StoreUint32(rdbw.ConnectionLostCounterAtomic, 0) + assert.Equal(t, 5*time.Second, rdbw.getConnectionCheckInterval()) + + //Should return 10s, if not connected and 4 <= counter < 8 + rdbw.CompareAndSetConnected(false) + atomic.StoreUint32(rdbw.ConnectionLostCounterAtomic, 4) + assert.Equal(t, 10*time.Second, rdbw.getConnectionCheckInterval()) + + //Should return 30s, if not connected and 8 <= counter < 11 + rdbw.CompareAndSetConnected(false) + atomic.StoreUint32(rdbw.ConnectionLostCounterAtomic, 8) + assert.Equal(t, 30*time.Second, rdbw.getConnectionCheckInterval()) + + //Should return 60s, if not connected and 11 <= counter < 14 + rdbw.CompareAndSetConnected(false) + atomic.StoreUint32(rdbw.ConnectionLostCounterAtomic, 11) + assert.Equal(t, 60*time.Second, rdbw.getConnectionCheckInterval()) + + //dbw.ConnectionLostCounter = 14 + //interval = dbw.getConnectionCheckInterval() + //TODO: Check for Fatal +} + +func TestRDBWrapper_CheckConnection(t *testing.T) { + rdbw := NewTestRDBW(nil) + + rdbw.Rdb = redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:6379", + DialTimeout: time.Minute / 2, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + }) + atomic.StoreUint32(rdbw.ConnectionLostCounterAtomic, 512312312) + assert.True(t, rdbw.CheckConnection(false), "DBWrapper should be connected") + assert.Equal(t, uint32(0), atomic.LoadUint32(rdbw.ConnectionLostCounterAtomic)) + + rdbw.Rdb = redis.NewClient(&redis.Options{ + Addr: "dasdasdasdasdasd:5123", + DialTimeout: time.Minute / 2, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + }) + atomic.StoreUint32(rdbw.ConnectionLostCounterAtomic, 0) + assert.False(t, rdbw.CheckConnection(false), "DBWrapper should not be connected") + assert.Equal(t, uint32(0), atomic.LoadUint32(rdbw.ConnectionLostCounterAtomic)) + + atomic.StoreUint32(rdbw.ConnectionLostCounterAtomic, 10) + assert.False(t, rdbw.CheckConnection(true), "DBWrapper should not be connected") + assert.Equal(t, uint32(11), atomic.LoadUint32(rdbw.ConnectionLostCounterAtomic)) +} From ca68b997d991a73779a2b14e4f0691970fb2ff42 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 15:44:07 +0100 Subject: [PATCH 37/69] Fix typo --- mysql_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql_test.go b/mysql_test.go index 92c2e0c7..e97ff4df 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -77,7 +77,7 @@ func TestNewDBWrapper(t *testing.T) { //TODO: Add more tests here } -func TestRDBWrapper_CheckConnection(t *testing.T) { +func TestDBWrapper_CheckConnection(t *testing.T) { mockDb := new(DbMock) dbw := NewTestDBW(mockDb) From debeb88de661a87fb5d00b4e7eeebfb027c81042 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 18:10:37 +0100 Subject: [PATCH 38/69] Fix imports --- mysql.go | 2 +- redis.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql.go b/mysql.go index d8f11733..dfeca6e7 100644 --- a/mysql.go +++ b/mysql.go @@ -4,7 +4,7 @@ import ( "container/list" "context" "database/sql" - "git.icinga.com/icingadb/icingadb-utils-lib" + "git.icinga.com/icingadb/icingadb-utils" log "github.com/sirupsen/logrus" "sync" "sync/atomic" diff --git a/redis.go b/redis.go index 331389a0..37854fac 100644 --- a/redis.go +++ b/redis.go @@ -1,7 +1,7 @@ package icingadb_connection import ( - "git.icinga.com/icingadb/icingadb-utils-lib" + "git.icinga.com/icingadb/icingadb-utils" "github.com/go-redis/redis" log "github.com/sirupsen/logrus" "sync" From 4f09b5b5e82a9244b2c43e29b98e375e1e1b82e1 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 18:11:00 +0100 Subject: [PATCH 39/69] Remove Redis testing mocks --- redis_test.go | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/redis_test.go b/redis_test.go index 2a699b04..00c5787c 100644 --- a/redis_test.go +++ b/redis_test.go @@ -3,52 +3,12 @@ package icingadb_connection import ( "github.com/go-redis/redis" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "sync" "sync/atomic" "testing" "time" ) -type RdbMock struct { - mock.Mock -} - -func (m *RdbMock) Ping() *redis.StatusCmd { - args := m.Called() - return args.Get(0).(*redis.StatusCmd) -} - -func (m *RdbMock) Publish(channel string, message interface{}) *redis.IntCmd { - args := m.Called(channel, message) - return args.Get(0).(*redis.IntCmd) -} - -func (m *RdbMock) XRead(a *redis.XReadArgs) *redis.XStreamSliceCmd { - args := m.Called(a) - return args.Get(0).(*redis.XStreamSliceCmd) -} - -func (m *RdbMock) XDel(stream string, ids ...string) *redis.IntCmd { - args := m.Called(stream, ids) - return args.Get(0).(*redis.IntCmd) -} - -func (m *RdbMock) HGetAll(key string) *redis.StringStringMapCmd { - args := m.Called(key) - return args.Get(0).(*redis.StringStringMapCmd) -} - -func (m *RdbMock) TxPipelined(fn func(redis.Pipeliner) error) ([]redis.Cmder, error) { - args := m.Called(fn) - return args.Get(0).([]redis.Cmder), args.Error(1) -} - -func (m *RdbMock) Subscribe(channels ...string) *redis.PubSub { - args := m.Called(channels) - return args.Get(0).(*redis.PubSub) -} - func NewTestRDBW(rdb RedisClient) RDBWrapper { dbw := RDBWrapper{Rdb: rdb, ConnectedAtomic: new(uint32), ConnectionLostCounterAtomic: new(uint32)} dbw.ConnectionUpCondition = sync.NewCond(&sync.Mutex{}) From 8f6d12c09ef416fcb3537b3a797e82d2adf77c0d Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 18:11:12 +0100 Subject: [PATCH 40/69] Redis: Add more tests --- redis_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/redis_test.go b/redis_test.go index 00c5787c..05e76308 100644 --- a/redis_test.go +++ b/redis_test.go @@ -83,3 +83,113 @@ func TestRDBWrapper_CheckConnection(t *testing.T) { assert.False(t, rdbw.CheckConnection(true), "DBWrapper should not be connected") assert.Equal(t, uint32(11), atomic.LoadUint32(rdbw.ConnectionLostCounterAtomic)) } + +func TestRDBWrapper_HGetAll(t *testing.T) { + rdb := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:6379", + DialTimeout: time.Minute / 2, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + }) + rdbw := NewTestRDBW(rdb) + + if !rdbw.CheckConnection(true) { + t.Fatal("This test needs a working Redis connection") + } + + rdb.Del("herpdaderp") + rdb.HSet("herpdaderp", "one", 5) + rdb.HSet("herpdaderp", "two", 11) + + rdbw.CompareAndSetConnected(false) + + var data map[string]string + var err error + done := make(chan bool) + go func() { + data, err = rdbw.HGetAll("herpdaderp") + done <- true + }() + + time.Sleep(500 * time.Millisecond) + rdbw.CheckConnection(true) + + <- done + + assert.NoError(t, err) + assert.Contains(t, data, "one") + assert.Contains(t, data, "two") +} + +func TestRDBWrapper_XRead(t *testing.T) { + rdb := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:6379", + DialTimeout: time.Minute / 2, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + }) + rdbw := NewTestRDBW(rdb) + + if !rdbw.CheckConnection(true) { + t.Fatal("This test needs a working Redis connection") + } + + rdb.XTrim("teststream", 0) + rdb.XAdd(&redis.XAddArgs{Stream: "teststream", Values: map[string]interface{}{"one": "5", "two": "11", "herp": "11"}}) + + rdbw.CompareAndSetConnected(false) + + var data *redis.XStreamSliceCmd + done := make(chan bool) + go func() { + data = rdbw.XRead(&redis.XReadArgs{Streams: []string{"teststream", "0"}}) + done <- true + }() + + time.Sleep(500 * time.Millisecond) + rdbw.CheckConnection(true) + + <- done + + streams, err := data.Result() + assert.NoError(t, err) + value := streams[0].Messages[0].Values + + assert.Contains(t, value, "one") + assert.Contains(t, value, "two") +} + +func TestRDBWrapper_XDel(t *testing.T) { + rdb := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:6379", + DialTimeout: time.Minute / 2, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + }) + rdbw := NewTestRDBW(rdb) + + if !rdbw.CheckConnection(true) { + t.Fatal("This test needs a working Redis connection") + } + + rdb.XTrim("teststream", 0) + adds := rdb.XAdd(&redis.XAddArgs{Stream: "teststream", Values: map[string]interface{}{"one": "5", "two": "11", "herp": "11"}}) + + rdbw.CompareAndSetConnected(false) + + done := make(chan bool) + go func() { + rdbw.XDel("teststream", adds.Val()) + done <- true + }() + + time.Sleep(500 * time.Millisecond) + rdbw.CheckConnection(true) + + <- done + + data := rdbw.XRead(&redis.XReadArgs{Streams: []string{"teststream", "0"}, Block: -1}) + streams, err := data.Result() + assert.Error(t, err) + assert.Len(t, streams, 0) +} From f2f43980987734bf56267ced8fbdc465c774a638 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 4 Mar 2019 18:11:20 +0100 Subject: [PATCH 41/69] Redis: Fix race condition --- redis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.go b/redis.go index 37854fac..4f7ee6a8 100644 --- a/redis.go +++ b/redis.go @@ -85,7 +85,7 @@ type RDBWrapper struct { } func (rdbw *RDBWrapper) IsConnected() bool { - return *rdbw.ConnectedAtomic != 0 + return atomic.LoadUint32(rdbw.ConnectedAtomic) != 0 } func (rdbw *RDBWrapper) CompareAndSetConnected(connected bool) (swapped bool) { From fed93f199d56acae1e783c3649a765fa88e72fbd Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 5 Mar 2019 10:39:54 +0100 Subject: [PATCH 42/69] Reduce test times --- mysql_test.go | 14 +++++++------- redis_test.go | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mysql_test.go b/mysql_test.go index e97ff4df..bcf2c60d 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -115,7 +115,7 @@ func TestDBWrapper_SqlCommit(t *testing.T) { done <- true }() - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Millisecond * 50) dbw.CompareAndSetConnected(true) dbw.ConnectionUpCondition.Broadcast() @@ -144,7 +144,7 @@ func TestDBWrapper_SqlBegin(t *testing.T) { done <- true }() - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Millisecond * 50) dbw.CompareAndSetConnected(true) dbw.ConnectionUpCondition.Broadcast() @@ -215,7 +215,7 @@ func TestDBWrapper_SqlQuery(t *testing.T) { done <- true }() - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Millisecond * 50) dbw.CompareAndSetConnected(true) dbw.ConnectionUpCondition.Broadcast() @@ -243,7 +243,7 @@ func TestDBWrapper_SqlExec(t *testing.T) { done <- true }() - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Millisecond * 50) dbw.CompareAndSetConnected(true) dbw.ConnectionUpCondition.Broadcast() @@ -271,7 +271,7 @@ func TestDBWrapper_SqlExecQuiet(t *testing.T) { done <- true }() - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Millisecond * 50) dbw.CompareAndSetConnected(true) dbw.ConnectionUpCondition.Broadcast() @@ -300,7 +300,7 @@ func TestDBWrapper_SqlExecTx(t *testing.T) { done <- true }() - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Millisecond * 50) dbw.CompareAndSetConnected(true) dbw.ConnectionUpCondition.Broadcast() @@ -330,7 +330,7 @@ func TestDBWrapper_SqlExecTxQuiet(t *testing.T) { done <- true }() - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Millisecond * 50) dbw.CompareAndSetConnected(true) dbw.ConnectionUpCondition.Broadcast() diff --git a/redis_test.go b/redis_test.go index 05e76308..272d1482 100644 --- a/redis_test.go +++ b/redis_test.go @@ -111,7 +111,7 @@ func TestRDBWrapper_HGetAll(t *testing.T) { done <- true }() - time.Sleep(500 * time.Millisecond) + time.Sleep(50 * time.Millisecond) rdbw.CheckConnection(true) <- done @@ -146,7 +146,7 @@ func TestRDBWrapper_XRead(t *testing.T) { done <- true }() - time.Sleep(500 * time.Millisecond) + time.Sleep(50 * time.Millisecond) rdbw.CheckConnection(true) <- done @@ -183,7 +183,7 @@ func TestRDBWrapper_XDel(t *testing.T) { done <- true }() - time.Sleep(500 * time.Millisecond) + time.Sleep(50 * time.Millisecond) rdbw.CheckConnection(true) <- done From ab08af7c3a80d7cb15d00e2f809f9cd05f0fab93 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 5 Mar 2019 10:40:19 +0100 Subject: [PATCH 43/69] Redis: Add more tests --- redis_pubsub_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ redis_test.go | 80 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 redis_pubsub_test.go diff --git a/redis_pubsub_test.go b/redis_pubsub_test.go new file mode 100644 index 00000000..34f6cbe9 --- /dev/null +++ b/redis_pubsub_test.go @@ -0,0 +1,74 @@ +package icingadb_connection + +import ( + "github.com/go-redis/redis" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestPubSubWrapper(t *testing.T) { + rdb := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:6379", + DialTimeout: time.Minute / 2, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + }) + rdbw := NewTestRDBW(rdb) + + if !rdbw.CheckConnection(true) { + t.Fatal("This test needs a working Redis connection") + } + + ps := rdbw.Subscribe() + + rdbw.CompareAndSetConnected(false) + + var errSubscribe error + done1:= make(chan bool) + go func () { + errSubscribe = ps.Subscribe("testchannel") + done1 <- true + }() + + time.Sleep(50 * time.Millisecond) + rdbw.CheckConnection(true) + + <- done1 + + rdbw.CompareAndSetConnected(false) + + var msg *redis.Message + var errReceive error + done2 := make(chan bool) + go func() { + msg, errReceive = ps.ReceiveMessage() + done2 <- true + }() + + time.Sleep(50 * time.Millisecond) + rdbw.CheckConnection(true) + + rdbw.Publish("testchannel", "Hello there") + + <- done2 + + rdbw.CompareAndSetConnected(false) + + var errClose error + done3:= make(chan bool) + go func () { + errClose = ps.Close() + done3 <- true + }() + + time.Sleep(50 * time.Millisecond) + rdbw.CheckConnection(true) + + <- done3 + + assert.NoError(t, errSubscribe) + assert.NoError(t, errReceive) + assert.NoError(t, errClose) + assert.Equal(t, "Hello there", msg.Payload) +} \ No newline at end of file diff --git a/redis_test.go b/redis_test.go index 272d1482..18810aab 100644 --- a/redis_test.go +++ b/redis_test.go @@ -193,3 +193,83 @@ func TestRDBWrapper_XDel(t *testing.T) { assert.Error(t, err) assert.Len(t, streams, 0) } + +func TestRDBWrapper_Publish(t *testing.T) { + rdb := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:6379", + DialTimeout: time.Minute / 2, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + }) + rdbw := NewTestRDBW(rdb) + + if !rdbw.CheckConnection(true) { + t.Fatal("This test needs a working Redis connection") + } + + var msg *redis.Message + var err error + done := make(chan bool) + go func() { + msg, err = rdb.Subscribe("testchannel").ReceiveMessage() + done <- true + }() + + rdbw.CompareAndSetConnected(false) + + go func () { + rdbw.Publish("testchannel", "Hello there") + }() + + time.Sleep(50 * time.Millisecond) + rdbw.CheckConnection(true) + + <- done + + assert.NoError(t, err) + assert.Equal(t, "Hello there", msg.Payload) +} + +func TestRDBWrapper_TxPipelined(t *testing.T) { + rdb := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:6379", + DialTimeout: time.Minute / 2, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + }) + rdbw := NewTestRDBW(rdb) + + if !rdbw.CheckConnection(true) { + t.Fatal("This test needs a working Redis connection") + } + + rdb.Del("firstKey") + rdb.Del("secondKey") + rdb.HSet("firstKey", "foo", 5) + rdb.HSet("secondKey", "bar", 11) + + rdbw.CompareAndSetConnected(false) + + var firstMap *redis.StringStringMapCmd + var secondMap *redis.StringStringMapCmd + var err error + done := make(chan bool) + go func() { + _, err = rdbw.TxPipelined(func(pipe redis.Pipeliner) error { + firstMap = pipe.HGetAll("firstKey") + secondMap = pipe.HGetAll("secondKey") + return nil + }) + done <- true + }() + + time.Sleep(50 * time.Millisecond) + rdbw.CheckConnection(true) + + <- done + + assert.NoError(t, err) + assert.Contains(t, firstMap.Val(), "foo") + assert.Contains(t, secondMap.Val(), "bar") + +} From f33c680685d2ea68ba65ce53cc1857d3aa687165 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 5 Mar 2019 13:39:12 +0100 Subject: [PATCH 44/69] Remove duplicate tests --- mysql_test.go | 88 --------------------------------------------------- 1 file changed, 88 deletions(-) diff --git a/mysql_test.go b/mysql_test.go index bcf2c60d..93b67ce4 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -254,94 +254,6 @@ func TestDBWrapper_SqlExec(t *testing.T) { mockDb.AssertExpectations(t) } -func TestDBWrapper_SqlExecQuiet(t *testing.T) { - mockDb := new(DbMock) - dbw := NewTestDBW(mockDb) - - mockDb.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, errors.New("whoops")).Once() - mockDb.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, nil).Once() - mockDb.On("Ping").Return(errors.New("whoops")).Once() - - var err error - done := make(chan bool) - - dbw.CompareAndSetConnected(true) - go func() { - _, err = dbw.SqlExecQuiet("test", "test") - done <- true - }() - - time.Sleep(time.Millisecond * 50) - - dbw.CompareAndSetConnected(true) - dbw.ConnectionUpCondition.Broadcast() - - <- done - - assert.NoError(t, err) - mockDb.AssertExpectations(t) -} - -func TestDBWrapper_SqlExecTx(t *testing.T) { - mockDb := new(DbMock) - dbw := NewTestDBW(mockDb) - mockTx := new(TransactionMock) - - mockTx.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, errors.New("whoops")).Once() - mockTx.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, nil).Once() - mockDb.On("Ping").Return(errors.New("whoops")).Once() - - var err error - done := make(chan bool) - - dbw.CompareAndSetConnected(true) - go func() { - _, err = dbw.SqlExecTx(mockTx, "test", "test") - done <- true - }() - - time.Sleep(time.Millisecond * 50) - - dbw.CompareAndSetConnected(true) - dbw.ConnectionUpCondition.Broadcast() - - <- done - - assert.NoError(t, err) - mockTx.AssertExpectations(t) - mockDb.AssertExpectations(t) -} - -func TestDBWrapper_SqlExecTxQuiet(t *testing.T) { - mockDb := new(DbMock) - dbw := NewTestDBW(mockDb) - mockTx := new(TransactionMock) - - mockTx.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, errors.New("whoops")).Once() - mockTx.On("Exec", "test", []interface{}(nil)).Return(SqlResultMock{}, nil).Once() - mockDb.On("Ping").Return(errors.New("whoops")).Once() - - var err error - done := make(chan bool) - - dbw.CompareAndSetConnected(true) - go func() { - _, err = dbw.SqlExecTxQuiet(mockTx, "test", "test") - done <- true - }() - - time.Sleep(time.Millisecond * 50) - - dbw.CompareAndSetConnected(true) - dbw.ConnectionUpCondition.Broadcast() - - <- done - - assert.NoError(t, err) - mockTx.AssertExpectations(t) - mockDb.AssertExpectations(t) -} - func TestGetConnectionCheckInterval(t *testing.T) { dbw := NewTestDBW(nil) From 0e8183b3fd32a86d696d8398cb922fab2d664ba9 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 5 Mar 2019 13:39:26 +0100 Subject: [PATCH 45/69] Add TestDBWrapper_SqlFetchAll() --- mysql_test.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/mysql_test.go b/mysql_test.go index 93b67ce4..d8dfb495 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -285,3 +285,34 @@ func TestGetConnectionCheckInterval(t *testing.T) { //interval = dbw.getConnectionCheckInterval() //TODO: Check for Fatal } + +func TestDBWrapper_SqlFetchAll(t *testing.T) { + dbw, err := NewDBWrapper("mysql", "module-dev:icinga0815!@tcp(127.0.0.1:3306)/icingadb") + assert.NoError(t, err, "Is the MySQL server running?") + + _, err = dbw.Db.Exec("CREATE TABLE testing0815 (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name varchar(255) NOT NULL)") + assert.NoError(t, err) + + _, err = dbw.Db.Exec("INSERT INTO testing0815 (name) VALUES ('horst'), ('test')") + assert.NoError(t, err) + + var res [][]interface{} + done := make(chan bool) + dbw.CompareAndSetConnected(false) + go func() { + res, err = dbw.SqlFetchAll("test", "SELECT * FROM testing0815") + done <- true + }() + + time.Sleep(time.Millisecond * 50) + + dbw.checkConnection(true) + + <- done + + assert.NoError(t, err) + assert.Equal(t, [][]interface {}([][]interface {}{{int64(1), "horst"}, {int64(2), "test"}}), res) + + _, err = dbw.Db.Exec("DROP TABLE testing0815") + assert.NoError(t, err) +} From aaf090ef78042f8b1ff10dcf1b71539b29791021 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 5 Mar 2019 13:50:02 +0100 Subject: [PATCH 46/69] Remove dbType from NewDBWrapper() because it's always MySQL --- mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql.go b/mysql.go index dfeca6e7..805b253a 100644 --- a/mysql.go +++ b/mysql.go @@ -29,8 +29,8 @@ type DbTransaction interface { Rollback() error } -func NewDBWrapper(dbType string, dbDsn string) (*DBWrapper, error) { - db, err := mkMysql(dbType, dbDsn) +func NewDBWrapper(dbDsn string) (*DBWrapper, error) { + db, err := mkMysql("mysql", dbDsn) if err != nil { return nil, err From dd7aca8c30993e02b33f36c5524ce74b0c6dbd0f Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 5 Mar 2019 17:09:00 +0100 Subject: [PATCH 47/69] Fix tests --- mysql_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql_test.go b/mysql_test.go index d8dfb495..b2ca129f 100644 --- a/mysql_test.go +++ b/mysql_test.go @@ -72,7 +72,7 @@ func NewTestDBW(db DbClient) DBWrapper { } func TestNewDBWrapper(t *testing.T) { - _, err := NewDBWrapper("mysql", "asdasd") + _, err := NewDBWrapper("asdasd") assert.Error(t, err) //TODO: Add more tests here } @@ -156,7 +156,7 @@ func TestDBWrapper_SqlBegin(t *testing.T) { } func TestDBWrapper_SqlTransaction(t *testing.T) { - dbw, err := NewDBWrapper("mysql", "module-dev:icinga0815!@tcp(127.0.0.1:3306)/icingadb") + dbw, err := NewDBWrapper( "module-dev:icinga0815!@tcp(127.0.0.1:3306)/icingadb") assert.NoError(t, err, "Is the MySQL server running?") err = dbw.SqlTransaction(false, true, false, func(tx DbTransaction) error { @@ -287,7 +287,7 @@ func TestGetConnectionCheckInterval(t *testing.T) { } func TestDBWrapper_SqlFetchAll(t *testing.T) { - dbw, err := NewDBWrapper("mysql", "module-dev:icinga0815!@tcp(127.0.0.1:3306)/icingadb") + dbw, err := NewDBWrapper("module-dev:icinga0815!@tcp(127.0.0.1:3306)/icingadb") assert.NoError(t, err, "Is the MySQL server running?") _, err = dbw.Db.Exec("CREATE TABLE testing0815 (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name varchar(255) NOT NULL)") From 972a717a32325151fda0e7307dcbf92bd211f37c Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 6 Mar 2019 09:09:21 +0100 Subject: [PATCH 48/69] Fix quiet transaction --- mysql.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mysql.go b/mysql.go index 805b253a..71390097 100644 --- a/mysql.go +++ b/mysql.go @@ -473,11 +473,8 @@ func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFai errTx := dbw.sqlTryTransaction(f, concurrencySafety, false) if !quiet { benchmarc.Stop() - } + DbIoSeconds.WithLabelValues("mysql", "transaction").Observe(benchmarc.Seconds()) - DbIoSeconds.WithLabelValues("mysql", "transaction").Observe(benchmarc.Seconds()) - - if !quiet { log.WithFields(log.Fields{ "context": "sql", "benchmark": benchmarc, From 5a5ef2c3d895f19cbdba1cc29201042d70d9a6ef Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 6 Mar 2019 16:29:18 +0100 Subject: [PATCH 49/69] Redis: Add HKeys() --- redis.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/redis.go b/redis.go index 4f7ee6a8..4421c816 100644 --- a/redis.go +++ b/redis.go @@ -67,6 +67,7 @@ type RedisClient interface { Publish(channel string, message interface{}) *redis.IntCmd XRead(a *redis.XReadArgs) *redis.XStreamSliceCmd XDel(stream string, ids ...string) *redis.IntCmd + HKeys(key string) *redis.StringSliceCmd HGetAll(key string) *redis.StringStringMapCmd TxPipelined(fn func(redis.Pipeliner) error) ([]redis.Cmder, error) Subscribe(channels ...string) *redis.PubSub @@ -220,6 +221,7 @@ func (rdbw *RDBWrapper) XRead(args *redis.XReadArgs) *redis.XStreamSliceCmd { return cmd } } + // Wrapper for connection handling func (rdbw *RDBWrapper) XDel(stream string, ids ...string) *redis.IntCmd { for { @@ -241,6 +243,27 @@ func (rdbw *RDBWrapper) XDel(stream string, ids ...string) *redis.IntCmd { } } +// Wrapper for connection handling +func (rdbw *RDBWrapper) HKeys(key string) *redis.StringSliceCmd { + for { + if !rdbw.IsConnected() { + rdbw.WaitForConnection() + continue + } + + cmd := rdbw.Rdb.HKeys(key) + _, err := cmd.Result() + + if err != nil { + if !rdbw.CheckConnection(false) { + continue + } + } + + return cmd + } +} + // Wrapper for auto-logging and connection handling func (rdbw *RDBWrapper) HGetAll(key string) (map[string]string, error) { for { From c2dd1a0fbcb23b03b3503b09ad5aed1700862794 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 6 Mar 2019 16:29:58 +0100 Subject: [PATCH 50/69] MySQL: Add row utils --- mysql_utils.go | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/mysql_utils.go b/mysql_utils.go index e991b247..eb150ac5 100644 --- a/mysql_utils.go +++ b/mysql_utils.go @@ -370,4 +370,74 @@ func isRetryableError(err error) bool { return true } return false +} + +var ( + bulkSize int +) + +func SetBulkSize(s int) { + bulkSize = s +} + +type Row interface { + InsertValues() []interface{} + UpdateValues() []interface{} + GetId() string + SetId(id string) +} + +type Rows []Row + +type RowFactory func() Row + +type BulkInsertStmt struct { + Format string + Fields []string + Placeholder string + NumField int +} + +func NewBulkInsertStmt(table string, fields []string) *BulkInsertStmt { + numField := len(fields) + placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(strings.Repeat("?, ", numField), ", ")) + stmt := BulkInsertStmt{ + Format: fmt.Sprintf("INSERT INTO %s (%s) VALUES %s ON DUPLICATE KEY UPDATE id = id", table, strings.Join(fields, ", "), "%s"), + Fields: fields, + Placeholder: placeholder, + NumField: numField, + } + + return &stmt +} + +type BulkDeleteStmt struct { + Format string +} + +func NewBulkDeleteStmt(table string) *BulkDeleteStmt { + stmt := BulkDeleteStmt{ + Format: fmt.Sprintf("DELETE FROM %s WHERE id IN (%s)", table, "%s"), + } + + return &stmt +} + +type UpdateStmt struct { + Statement string + NumField int +} + +func NewUpdateStmt(table string, fields []string) *UpdateStmt { + assignmentList := make([]string, len(fields)) + + for i, field := range fields { + assignmentList[i] = fmt.Sprintf("%s = ?", field) + } + stmt := UpdateStmt{ + Statement: fmt.Sprintf("UPDATE %s SET %s WHERE id = ?", table, strings.Join(assignmentList, ", ")), + NumField: len(fields)+1, // +1 because of the WHERE clause + } + + return &stmt } \ No newline at end of file From 6141ec4f1445505c10ed8516c652a21807cf3ea3 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 6 Mar 2019 16:30:08 +0100 Subject: [PATCH 51/69] MySQL: SqlFetchIds --- mysql.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/mysql.go b/mysql.go index 71390097..60e45b9d 100644 --- a/mysql.go +++ b/mysql.go @@ -4,6 +4,7 @@ import ( "container/list" "context" "database/sql" + "fmt" "git.icinga.com/icingadb/icingadb-utils" log "github.com/sirupsen/logrus" "sync" @@ -525,4 +526,42 @@ func (dbw *DBWrapper) sqlTryTransaction(f func(transaction DbTransaction) error, } return dbw.SqlCommit(tx, quiet) +} + +func (dbw *DBWrapper) SqlFetchIds(table string) ([]string, error) { + var keys []string + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + rows, err := dbw.SqlQuery(fmt.Sprintf("SELECT id FROM %s", table)) + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + } + + defer rows.Close() + + for rows.Next() { + var id []byte + + err = rows.Scan(&id) + if err != nil { + return nil, err + } + + keys = append(keys, icingadb_utils.DecodeChecksum(id)) + } + + err = rows.Err() + if err != nil { + return nil, err + } + + return keys, nil + } } \ No newline at end of file From d08c7750aa0862194a7342248adf2bafaba5bdf8 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 7 Mar 2019 17:19:31 +0100 Subject: [PATCH 52/69] Move environemtn to HA --- redis.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/redis.go b/redis.go index 4421c816..22943c4d 100644 --- a/redis.go +++ b/redis.go @@ -9,11 +9,6 @@ import ( "time" ) -type Environment struct { - ID []byte - Name string -} - type Icinga2RedisWriterEventsConfig struct { Update, Delete, Dump string } @@ -74,15 +69,14 @@ type RedisClient interface { } type StatusCmd interface { - } // Redis wrapper including helper functions type RDBWrapper struct { - Rdb RedisClient - ConnectedAtomic *uint32 //uint32 to be able to use atomic operations - ConnectionUpCondition *sync.Cond - ConnectionLostCounterAtomic *uint32 //uint32 to be able to use atomic operations + Rdb RedisClient + ConnectedAtomic *uint32 //uint32 to be able to use atomic operations + ConnectionUpCondition *sync.Cond + ConnectionLostCounterAtomic *uint32 //uint32 to be able to use atomic operations } func (rdbw *RDBWrapper) IsConnected() bool { @@ -108,7 +102,7 @@ func NewRDBWrapper(address string) (*RDBWrapper, error) { rdbw := RDBWrapper{ Rdb: rdb, ConnectedAtomic: new(uint32), ConnectionLostCounterAtomic: new(uint32), - ConnectionUpCondition: sync.NewCond(&sync.Mutex{}), + ConnectionUpCondition: sync.NewCond(&sync.Mutex{}), } _, err := rdbw.Rdb.Ping().Result() @@ -331,4 +325,4 @@ func (rdbw *RDBWrapper) Subscribe() PubSubWrapper { ps := rdbw.Rdb.Subscribe() psw := PubSubWrapper{ps: ps, rdbw: rdbw} return psw -} \ No newline at end of file +} From 8ab7a1a91d2cfd4c04855a40d16e44dd654b0424 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 8 Mar 2019 10:38:18 +0100 Subject: [PATCH 53/69] Move row back to main/configobject package --- mysql_utils.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mysql_utils.go b/mysql_utils.go index eb150ac5..c7d868aa 100644 --- a/mysql_utils.go +++ b/mysql_utils.go @@ -380,17 +380,6 @@ func SetBulkSize(s int) { bulkSize = s } -type Row interface { - InsertValues() []interface{} - UpdateValues() []interface{} - GetId() string - SetId(id string) -} - -type Rows []Row - -type RowFactory func() Row - type BulkInsertStmt struct { Format string Fields []string From 4f64bd91842a947ca721f9a0ad5e1dec3518e780 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 8 Mar 2019 16:40:34 +0100 Subject: [PATCH 54/69] Correct variable names and return values --- redis.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/redis.go b/redis.go index 22943c4d..15d3928a 100644 --- a/redis.go +++ b/redis.go @@ -259,7 +259,7 @@ func (rdbw *RDBWrapper) HKeys(key string) *redis.StringSliceCmd { } // Wrapper for auto-logging and connection handling -func (rdbw *RDBWrapper) HGetAll(key string) (map[string]string, error) { +func (rdbw *RDBWrapper) HGetAll(key string) *redis.StringStringMapCmd { for { if !rdbw.IsConnected() { rdbw.WaitForConnection() @@ -267,9 +267,9 @@ func (rdbw *RDBWrapper) HGetAll(key string) (map[string]string, error) { } benchmarc := icingadb_utils.NewBenchmark() - res, errHGA := rdbw.Rdb.HGetAll(key).Result() + res := rdbw.Rdb.HGetAll(key) - if errHGA != nil { + if _, err := res.Result(); err != nil { if !rdbw.CheckConnection(false) { continue } @@ -283,10 +283,10 @@ func (rdbw *RDBWrapper) HGetAll(key string) (map[string]string, error) { "context": "redis", "benchmark": benchmarc, "query": "HGETALL " + key, - "result": res, + "result": res.Val(), }).Debug("Ran Query") - return res, errHGA + return res } } @@ -299,9 +299,9 @@ func (rdbw *RDBWrapper) TxPipelined(fn func(pipeliner redis.Pipeliner) error) ([ } benchmarc := icingadb_utils.NewBenchmark() - c, e := rdbw.Rdb.TxPipelined(fn) + cmd, err := rdbw.Rdb.TxPipelined(fn) - if e != nil { + if err != nil { if !rdbw.CheckConnection(false) { continue } @@ -317,7 +317,7 @@ func (rdbw *RDBWrapper) TxPipelined(fn func(pipeliner redis.Pipeliner) error) ([ "query": "MULTI/EXEC", }).Debug("Ran pipelined transaction") - return c, e + return cmd, err } } From 48f438c7e64e27676db260e08fdf5c3ae893d6fc Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 12 Mar 2019 12:48:17 +0100 Subject: [PATCH 55/69] Add SqlBulkInsert --- mysql.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/mysql.go b/mysql.go index 60e45b9d..6a96d5c1 100644 --- a/mysql.go +++ b/mysql.go @@ -5,8 +5,10 @@ import ( "context" "database/sql" "fmt" + "git.icinga.com/icingadb/icingadb-main/configobject" "git.icinga.com/icingadb/icingadb-utils" log "github.com/sirupsen/logrus" + "strings" "sync" "sync/atomic" "time" @@ -564,4 +566,33 @@ func (dbw *DBWrapper) SqlFetchIds(table string) ([]string, error) { return keys, nil } +} + +func (dbw *DBWrapper) SqlBulkInsert(rows []configobject.Row, stmt *BulkInsertStmt) { + if len(rows) == 0 { + return + } + + placeholders := make([]string, len(rows)) + values := make([]interface{}, len(rows)*stmt.NumField) + j := 0 + + for i, r := range rows { + placeholders[i] = stmt.Placeholder + + for _, v := range r.InsertValues() { + values[j] = v + j++ + } + } + + query := fmt.Sprintf(stmt.Format, strings.Join(placeholders, ", ")) + + _, err := dbw.SqlExec("Bulk insert", query, values...) + if err != nil { + _, err = dbw.SqlExec("Bulk insert", query, values...) + if err != nil { + panic(err) + } + } } \ No newline at end of file From 7f02a6a1b2a7d0c6cc0128b6f8b421cf8ded0c4e Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 12 Mar 2019 12:48:47 +0100 Subject: [PATCH 56/69] Add Redis Pipeliner Wrapper --- redis.go | 7 +++++++ redis_pipeliner.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 redis_pipeliner.go diff --git a/redis.go b/redis.go index 15d3928a..18287609 100644 --- a/redis.go +++ b/redis.go @@ -65,6 +65,7 @@ type RedisClient interface { HKeys(key string) *redis.StringSliceCmd HGetAll(key string) *redis.StringStringMapCmd TxPipelined(fn func(redis.Pipeliner) error) ([]redis.Cmder, error) + Pipeline() redis.Pipeliner Subscribe(channels ...string) *redis.PubSub } @@ -321,6 +322,12 @@ func (rdbw *RDBWrapper) TxPipelined(fn func(pipeliner redis.Pipeliner) error) ([ } } +func (rdbw *RDBWrapper) Pipeline() PipelinerWrapper { + pipeliner := rdbw.Rdb.Pipeline() + plw := PipelinerWrapper{pipeliner: pipeliner, rdbw: rdbw} + return plw +} + func (rdbw *RDBWrapper) Subscribe() PubSubWrapper { ps := rdbw.Rdb.Subscribe() psw := PubSubWrapper{ps: ps, rdbw: rdbw} diff --git a/redis_pipeliner.go b/redis_pipeliner.go new file mode 100644 index 00000000..97e2fbbc --- /dev/null +++ b/redis_pipeliner.go @@ -0,0 +1,47 @@ +package icingadb_connection + +import "github.com/go-redis/redis" + +type PipelinerWrapper struct { + pipeliner redis.Pipeliner + rdbw *RDBWrapper +} + +func (plw *PipelinerWrapper) Exec() ([]redis.Cmder, error) { + for { + if !plw.rdbw.IsConnected() { + plw.rdbw.WaitForConnection() + continue + } + + cmder, err := plw.pipeliner.Exec() + + if err != nil { + if !plw.rdbw.CheckConnection(false) { + continue + } + } + + return cmder, err + } +} + +func (plw *PipelinerWrapper) HMGet(key string, fields ...string) *redis.SliceCmd { + for { + if !plw.rdbw.IsConnected() { + plw.rdbw.WaitForConnection() + continue + } + + cmd := plw.pipeliner.HMGet(key, fields...) + _, err := cmd.Result() + + if err != nil { + if !plw.rdbw.CheckConnection(false) { + continue + } + } + + return cmd + } +} \ No newline at end of file From 5df2040e974149407dcc8f3568202478eb490de8 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 12 Mar 2019 12:49:01 +0100 Subject: [PATCH 57/69] Add PipeConfigChunks --- redis.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/redis.go b/redis.go index 18287609..d723b09f 100644 --- a/redis.go +++ b/redis.go @@ -1,6 +1,7 @@ package icingadb_connection import ( + "fmt" "git.icinga.com/icingadb/icingadb-utils" "github.com/go-redis/redis" log "github.com/sirupsen/logrus" @@ -333,3 +334,64 @@ func (rdbw *RDBWrapper) Subscribe() PubSubWrapper { psw := PubSubWrapper{ps: ps, rdbw: rdbw} return psw } + +type ConfigChunk struct { + Keys []string + Configs []interface{} + Checksums []interface{} +} + +func (rdbw *RDBWrapper) PipeConfigChunks(done <-chan struct{}, keys []string, objectType string) <-chan *ConfigChunk { + out := make(chan *ConfigChunk) + + worker := func(chunk <-chan []string) { + for k := range chunk { + pipe := rdbw.Pipeline() + cmds := make([]*redis.SliceCmd, 2) + + cmds[0] = pipe.HMGet(fmt.Sprintf("icinga:config:object:%s", objectType), k...) + cmds[1] = pipe.HMGet(fmt.Sprintf("icinga:config:checksum:%s", objectType), k...) + + _, err := pipe.Exec() // TODO(el): What to do with the Cmder slice? + if err != nil { + panic(err) + } + + configs, err := cmds[0].Result() + if err != nil { + panic(err) + } + checksums, err := cmds[1].Result() + if err != nil { + panic(err) + } + + select { + case out <- &ConfigChunk{Keys: k, Configs: configs, Checksums: checksums}: + case <-done: + return + } + } + } + + //TODO: Replace fixed chunkSize + work := icingadb_utils.ChunkKeys(done, keys, 1500) + + go func() { + defer close(out) + + wg := &sync.WaitGroup{} + + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + worker(work) + }() + } + + wg.Wait() + }() + + return out +} From 380e6d40fc8c151a14eb99f92ab3299a347e387d Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 13 Mar 2019 11:20:45 +0100 Subject: [PATCH 58/69] SqlBulkInsert should return an error --- mysql.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mysql.go b/mysql.go index 6a96d5c1..0cca6b26 100644 --- a/mysql.go +++ b/mysql.go @@ -568,9 +568,9 @@ func (dbw *DBWrapper) SqlFetchIds(table string) ([]string, error) { } } -func (dbw *DBWrapper) SqlBulkInsert(rows []configobject.Row, stmt *BulkInsertStmt) { +func (dbw *DBWrapper) SqlBulkInsert(rows []configobject.Row, stmt *BulkInsertStmt) error { if len(rows) == 0 { - return + return nil } placeholders := make([]string, len(rows)) @@ -590,7 +590,11 @@ func (dbw *DBWrapper) SqlBulkInsert(rows []configobject.Row, stmt *BulkInsertStm _, err := dbw.SqlExec("Bulk insert", query, values...) if err != nil { - _, err = dbw.SqlExec("Bulk insert", query, values...) + return err + } + + return nil +} if err != nil { panic(err) } From ddd64e7504f750eaea9c65023c5c013ce6fd3ca9 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 13 Mar 2019 11:21:05 +0100 Subject: [PATCH 59/69] Add SqlBulkDelete --- mysql.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/mysql.go b/mysql.go index 0cca6b26..35398b25 100644 --- a/mysql.go +++ b/mysql.go @@ -595,8 +595,31 @@ func (dbw *DBWrapper) SqlBulkInsert(rows []configobject.Row, stmt *BulkInsertStm return nil } + +func (dbw *DBWrapper) SqlBulkDelete(keys []string, stmt *BulkDeleteStmt) error { + if len(keys) == 0 { + return nil + } + + done := make(chan struct{}) + defer close(done) + + //TODO: Don't do this hardcoded - Chunksize + for bulk := range icingadb_utils.ChunkKeys(done, keys, 1000) { + placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(bulk)), ", ") + values := make([]interface{}, len(bulk)) + + for i, key := range bulk { + values[i] = key + } + + query := fmt.Sprintf(stmt.Format, placeholders) + + _, err := dbw.SqlExec("Bulk insert", query, values...) if err != nil { - panic(err) + return err } } + + return nil } \ No newline at end of file From e6eb55207f5c900840a98e8a2c21d98e1caac469 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 15 Mar 2019 15:58:41 +0100 Subject: [PATCH 60/69] Redis: Add HMGet() --- redis.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/redis.go b/redis.go index d723b09f..63c57506 100644 --- a/redis.go +++ b/redis.go @@ -64,6 +64,7 @@ type RedisClient interface { XRead(a *redis.XReadArgs) *redis.XStreamSliceCmd XDel(stream string, ids ...string) *redis.IntCmd HKeys(key string) *redis.StringSliceCmd + HMGet(key string, fields ...string) *redis.SliceCmd HGetAll(key string) *redis.StringStringMapCmd TxPipelined(fn func(redis.Pipeliner) error) ([]redis.Cmder, error) Pipeline() redis.Pipeliner @@ -260,6 +261,26 @@ func (rdbw *RDBWrapper) HKeys(key string) *redis.StringSliceCmd { } } +func (rdbw * RDBWrapper) HMGet(key string, fields ...string) *redis.SliceCmd { + for { + if !rdbw.IsConnected() { + rdbw.WaitForConnection() + continue + } + + cmd := rdbw.Rdb.HMGet(key, fields...) + _, err := cmd.Result() + + if err != nil { + if !rdbw.CheckConnection(false) { + continue + } + } + + return cmd + } +} + // Wrapper for auto-logging and connection handling func (rdbw *RDBWrapper) HGetAll(key string) *redis.StringStringMapCmd { for { From 8fd8758620f3c107127afee2c3b84f832855e0e3 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 15 Mar 2019 15:59:14 +0100 Subject: [PATCH 61/69] Add SqlFetchChecksums() & SqlBulkUpdate() --- mysql.go | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/mysql.go b/mysql.go index 35398b25..8cdd1cde 100644 --- a/mysql.go +++ b/mysql.go @@ -544,6 +544,8 @@ func (dbw *DBWrapper) SqlFetchIds(table string) ([]string, error) { if !dbw.checkConnection(false) { continue } + + return nil, err } defer rows.Close() @@ -568,6 +570,50 @@ func (dbw *DBWrapper) SqlFetchIds(table string) ([]string, error) { } } +func (dbw *DBWrapper) SqlFetchChecksums(table string, ids []string) (map[string]map[string]string, error) { + var checksums = map[string]map[string]string{} + for { + if !dbw.IsConnected() { + dbw.WaitForConnection() + continue + } + + query := fmt.Sprintf("SELECT id, properties_checksum FROM %s WHERE id IN (X'%s')", table, strings.Join(ids, "', X'")) + rows, err := dbw.SqlQuery(query) + + if err != nil { + if !dbw.checkConnection(false) { + continue + } + + return nil, err + } + + defer rows.Close() + + for rows.Next() { + var id []byte + var propertiesChecksum []byte + + err = rows.Scan(&id, &propertiesChecksum) + if err != nil { + return nil, err + } + + checksums[icingadb_utils.DecodeChecksum(id)] = map[string]string{ + "properties_checksum": icingadb_utils.DecodeChecksum(propertiesChecksum), + } + } + + err = rows.Err() + if err != nil { + return nil, err + } + + return checksums, nil + } +} + func (dbw *DBWrapper) SqlBulkInsert(rows []configobject.Row, stmt *BulkInsertStmt) error { if len(rows) == 0 { return nil @@ -610,16 +656,47 @@ func (dbw *DBWrapper) SqlBulkDelete(keys []string, stmt *BulkDeleteStmt) error { values := make([]interface{}, len(bulk)) for i, key := range bulk { - values[i] = key + values[i] = icingadb_utils.Checksum(key) } - query := fmt.Sprintf(stmt.Format, placeholders) - _, err := dbw.SqlExec("Bulk insert", query, values...) + _, err := dbw.SqlExec("Bulk delete", query, values...) if err != nil { return err } } return nil +} + +func (dbw *DBWrapper) SqlBulkUpdate(rows []configobject.Row, statement *UpdateStmt) error { + if len(rows) == 0 { + return nil + } + + values := make([]interface{}, statement.NumField) + err := dbw.SqlTransaction(true, false, false, func(tx DbTransaction) error { + for _, r := range rows { + var ( + i = 0 + v interface{} + ) + + for i, v = range r.UpdateValues() { + values[i] = v + } + + i++ + values[i] = icingadb_utils.Checksum(r.GetId()) + + _, err := tx.Exec(statement.Statement, values...) + if err != nil { + return err + } + } + + return nil + }) + + return err } \ No newline at end of file From 588b339dc41bd24ea82ee3ad0c04c9f97ebe0dbc Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 15 Mar 2019 15:59:41 +0100 Subject: [PATCH 62/69] Add PipeChecksumChunks() --- redis.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/redis.go b/redis.go index 63c57506..d71700b6 100644 --- a/redis.go +++ b/redis.go @@ -362,6 +362,11 @@ type ConfigChunk struct { Checksums []interface{} } +type ChecksumChunk struct { + Keys []string + Checksums []interface{} +} + func (rdbw *RDBWrapper) PipeConfigChunks(done <-chan struct{}, keys []string, objectType string) <-chan *ConfigChunk { out := make(chan *ConfigChunk) @@ -396,7 +401,7 @@ func (rdbw *RDBWrapper) PipeConfigChunks(done <-chan struct{}, keys []string, ob } //TODO: Replace fixed chunkSize - work := icingadb_utils.ChunkKeys(done, keys, 1500) + work := icingadb_utils.ChunkKeys(done, keys, 500) go func() { defer close(out) @@ -416,3 +421,46 @@ func (rdbw *RDBWrapper) PipeConfigChunks(done <-chan struct{}, keys []string, ob return out } + +func (rdbw *RDBWrapper) PipeChecksumChunks(done <-chan struct{}, keys []string, objectType string) <-chan *ChecksumChunk { + out := make(chan *ChecksumChunk) + + worker := func(chunk <-chan []string) { + for k := range chunk { + cmd := rdbw.HMGet(fmt.Sprintf("icinga:config:checksum:%s", objectType), k...) + + checksums, err := cmd.Result() + if err != nil { + panic(err) + } + + select { + case out <- &ChecksumChunk{Keys: k, Checksums: checksums}: + case <-done: + return + } + } + } + + //TODO: Replace fixed chunkSize + work := icingadb_utils.ChunkKeys(done, keys, 500) + + go func() { + defer close(out) + + wg := &sync.WaitGroup{} + + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + worker(work) + }() + } + + wg.Wait() + }() + + return out +} + From e2563ee2a4cffafc81c88a61aaa0e521c934526c Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 15 Mar 2019 16:32:13 +0100 Subject: [PATCH 63/69] SqlFetchChecksums: Fetch in chunks --- mysql.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/mysql.go b/mysql.go index 8cdd1cde..483f4aad 100644 --- a/mysql.go +++ b/mysql.go @@ -572,13 +572,12 @@ func (dbw *DBWrapper) SqlFetchIds(table string) ([]string, error) { func (dbw *DBWrapper) SqlFetchChecksums(table string, ids []string) (map[string]map[string]string, error) { var checksums = map[string]map[string]string{} - for { - if !dbw.IsConnected() { - dbw.WaitForConnection() - continue - } - query := fmt.Sprintf("SELECT id, properties_checksum FROM %s WHERE id IN (X'%s')", table, strings.Join(ids, "', X'")) + done := make(chan struct{}) + //TODO: Don't do this hardcoded - Chunksize + for bulk := range icingadb_utils.ChunkKeys(done, ids, 1000) { + //TODO: This should be done in parallel + query := fmt.Sprintf("SELECT id, properties_checksum FROM %s WHERE id IN (X'%s')", table, strings.Join(bulk, "', X'")) rows, err := dbw.SqlQuery(query) if err != nil { @@ -609,9 +608,9 @@ func (dbw *DBWrapper) SqlFetchChecksums(table string, ids []string) (map[string] if err != nil { return nil, err } - - return checksums, nil } + + return checksums, nil } func (dbw *DBWrapper) SqlBulkInsert(rows []configobject.Row, stmt *BulkInsertStmt) error { From 4bb3a18a3c75d3a4a5ffc6870b215dc44a7b5132 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 19 Mar 2019 10:41:06 +0100 Subject: [PATCH 64/69] Replace UpdateStmt with BulkUpdateStmt and use "REPLACE INTO" --- mysql.go | 38 +++++++++++++++++--------------------- mysql_utils.go | 33 +++++++++++++-------------------- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/mysql.go b/mysql.go index 483f4aad..dda19fd6 100644 --- a/mysql.go +++ b/mysql.go @@ -668,34 +668,30 @@ func (dbw *DBWrapper) SqlBulkDelete(keys []string, stmt *BulkDeleteStmt) error { return nil } -func (dbw *DBWrapper) SqlBulkUpdate(rows []configobject.Row, statement *UpdateStmt) error { +func (dbw *DBWrapper) SqlBulkUpdate(rows []configobject.Row, stmt *BulkUpdateStmt) error { if len(rows) == 0 { return nil } - values := make([]interface{}, statement.NumField) - err := dbw.SqlTransaction(true, false, false, func(tx DbTransaction) error { - for _, r := range rows { - var ( - i = 0 - v interface{} - ) + placeholders := make([]string, len(rows)) + values := make([]interface{}, len(rows)*stmt.NumField) + j := 0 - for i, v = range r.UpdateValues() { - values[i] = v - } + for i, r := range rows { + placeholders[i] = stmt.Placeholder - i++ - values[i] = icingadb_utils.Checksum(r.GetId()) - - _, err := tx.Exec(statement.Statement, values...) - if err != nil { - return err - } + for _, v := range r.InsertValues() { + values[j] = v + j++ } + } - return nil - }) + query := fmt.Sprintf(stmt.Format, strings.Join(placeholders, ", ")) - return err + _, err := dbw.SqlExec("Bulk insert", query, values...) + if err != nil { + return err + } + + return nil } \ No newline at end of file diff --git a/mysql_utils.go b/mysql_utils.go index c7d868aa..d596e510 100644 --- a/mysql_utils.go +++ b/mysql_utils.go @@ -372,14 +372,6 @@ func isRetryableError(err error) bool { return false } -var ( - bulkSize int -) - -func SetBulkSize(s int) { - bulkSize = s -} - type BulkInsertStmt struct { Format string Fields []string @@ -412,20 +404,21 @@ func NewBulkDeleteStmt(table string) *BulkDeleteStmt { return &stmt } -type UpdateStmt struct { - Statement string - NumField int +type BulkUpdateStmt struct { + Format string + Fields []string + Placeholder string + NumField int } -func NewUpdateStmt(table string, fields []string) *UpdateStmt { - assignmentList := make([]string, len(fields)) - - for i, field := range fields { - assignmentList[i] = fmt.Sprintf("%s = ?", field) - } - stmt := UpdateStmt{ - Statement: fmt.Sprintf("UPDATE %s SET %s WHERE id = ?", table, strings.Join(assignmentList, ", ")), - NumField: len(fields)+1, // +1 because of the WHERE clause +func NewBulkUpdateStmt(table string, fields []string) *BulkUpdateStmt { + numField := len(fields) + placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(strings.Repeat("?, ", numField), ", ")) + stmt := BulkUpdateStmt{ + Format: fmt.Sprintf("REPLACE INTO %s (%s) VALUES %s", table, strings.Join(fields, ", "), "%s"), + Fields: fields, + Placeholder: placeholder, + NumField: numField, } return &stmt From 2265c22041f5556b4ea5e1f6e140537f9c979460 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 21 Mar 2019 10:14:22 +0100 Subject: [PATCH 65/69] Move Prometheus HTTPD to IcingaDB-Main --- prometheus.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/prometheus.go b/prometheus.go index f977c88a..8b9dc482 100644 --- a/prometheus.go +++ b/prometheus.go @@ -3,9 +3,6 @@ package icingadb_connection import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/client_golang/prometheus/promhttp" - log "github.com/sirupsen/logrus" - "net/http" ) var DbIoSeconds = promauto.NewSummaryVec( @@ -30,10 +27,3 @@ var DbOperationsExec = promauto.NewCounter(prometheus.CounterOpts{ Name: "db_operations_exec", Help: "Database exec operations since startup", }) - -//TODO: Move this to main package of IcingaDB -func Httpd(addr string, chErr chan error) { - http.Handle("/metrics", promhttp.Handler()) - log.Infof("Serving debug info at http://%s/metrics", addr) - chErr <- http.ListenAndServe(addr, nil) -} From 698a6379f5249a8f496d1897cddaebec39be7fb4 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 22 Mar 2019 11:22:04 +0100 Subject: [PATCH 66/69] SqlFetchIds(): Use envId --- mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql.go b/mysql.go index dda19fd6..b49aa427 100644 --- a/mysql.go +++ b/mysql.go @@ -530,7 +530,7 @@ func (dbw *DBWrapper) sqlTryTransaction(f func(transaction DbTransaction) error, return dbw.SqlCommit(tx, quiet) } -func (dbw *DBWrapper) SqlFetchIds(table string) ([]string, error) { +func (dbw *DBWrapper) SqlFetchIds(envId []byte, table string) ([]string, error) { var keys []string for { if !dbw.IsConnected() { @@ -538,7 +538,7 @@ func (dbw *DBWrapper) SqlFetchIds(table string) ([]string, error) { continue } - rows, err := dbw.SqlQuery(fmt.Sprintf("SELECT id FROM %s", table)) + rows, err := dbw.SqlQuery(fmt.Sprintf("SELECT id FROM %s WHERE env_id='X%s'", table, icingadb_utils.DecodeChecksum(envId))) if err != nil { if !dbw.checkConnection(false) { From 07b32ecaf742b2309f302727abded1e11165117c Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 22 Mar 2019 15:00:15 +0100 Subject: [PATCH 67/69] Retry on SQL deadlock --- mysql.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mysql.go b/mysql.go index b49aa427..312f4a53 100644 --- a/mysql.go +++ b/mysql.go @@ -633,7 +633,10 @@ func (dbw *DBWrapper) SqlBulkInsert(rows []configobject.Row, stmt *BulkInsertStm query := fmt.Sprintf(stmt.Format, strings.Join(placeholders, ", ")) - _, err := dbw.SqlExec("Bulk insert", query, values...) + _, err := dbw.WithRetry(func() (result sql.Result, e error) { + return dbw.SqlExec("Bulk insert", query, values...) + }) + if err != nil { return err } @@ -659,7 +662,9 @@ func (dbw *DBWrapper) SqlBulkDelete(keys []string, stmt *BulkDeleteStmt) error { } query := fmt.Sprintf(stmt.Format, placeholders) - _, err := dbw.SqlExec("Bulk delete", query, values...) + _, err := dbw.WithRetry(func() (result sql.Result, e error) { + return dbw.SqlExec("Bulk delete", query, values...) + }) if err != nil { return err } @@ -688,7 +693,9 @@ func (dbw *DBWrapper) SqlBulkUpdate(rows []configobject.Row, stmt *BulkUpdateStm query := fmt.Sprintf(stmt.Format, strings.Join(placeholders, ", ")) - _, err := dbw.SqlExec("Bulk insert", query, values...) + _, err := dbw.WithRetry(func() (result sql.Result, e error) { + return dbw.SqlExec("Bulk update", query, values...) + }) if err != nil { return err } From b0ce152219dcc0904f42879223423d517fdbd8d3 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 22 Mar 2019 15:25:52 +0100 Subject: [PATCH 68/69] Fix env --- mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql.go b/mysql.go index 312f4a53..3514c67b 100644 --- a/mysql.go +++ b/mysql.go @@ -538,7 +538,7 @@ func (dbw *DBWrapper) SqlFetchIds(envId []byte, table string) ([]string, error) continue } - rows, err := dbw.SqlQuery(fmt.Sprintf("SELECT id FROM %s WHERE env_id='X%s'", table, icingadb_utils.DecodeChecksum(envId))) + rows, err := dbw.SqlQuery(fmt.Sprintf("SELECT id FROM %s WHERE env_id=(X'%s')", table, icingadb_utils.DecodeChecksum(envId))) if err != nil { if !dbw.checkConnection(false) { From 9a5136190f2c2cf2d2aec56aee653809001943cb Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 13 May 2019 14:40:51 +0200 Subject: [PATCH 69/69] Prepare repository merge --- .gitignore => connection/.gitignore | 0 connection/.gitlab-ci.yml | 30 +++++++++++++++++++ README.md => connection/README.md | 0 coverage.sh => connection/coverage.sh | 0 mysql.go => connection/mysql.go | 2 +- mysql_test.go => connection/mysql_test.go | 2 +- mysql_utils.go => connection/mysql_utils.go | 4 +-- .../mysql_utils_test.go | 2 +- prometheus.go => connection/prometheus.go | 2 +- redis.go => connection/redis.go | 2 +- .../redis_pipeliner.go | 2 +- redis_pubsub.go => connection/redis_pubsub.go | 2 +- .../redis_pubsub_test.go | 2 +- redis_test.go => connection/redis_test.go | 2 +- test_db.sql => connection/test_db.sql | 0 15 files changed, 41 insertions(+), 11 deletions(-) rename .gitignore => connection/.gitignore (100%) create mode 100644 connection/.gitlab-ci.yml rename README.md => connection/README.md (100%) rename coverage.sh => connection/coverage.sh (100%) rename mysql.go => connection/mysql.go (99%) rename mysql_test.go => connection/mysql_test.go (99%) rename mysql_utils.go => connection/mysql_utils.go (99%) rename mysql_utils_test.go => connection/mysql_utils_test.go (98%) rename prometheus.go => connection/prometheus.go (96%) rename redis.go => connection/redis.go (99%) rename redis_pipeliner.go => connection/redis_pipeliner.go (96%) rename redis_pubsub.go => connection/redis_pubsub.go (97%) rename redis_pubsub_test.go => connection/redis_pubsub_test.go (97%) rename redis_test.go => connection/redis_test.go (99%) rename test_db.sql => connection/test_db.sql (100%) diff --git a/.gitignore b/connection/.gitignore similarity index 100% rename from .gitignore rename to connection/.gitignore diff --git a/connection/.gitlab-ci.yml b/connection/.gitlab-ci.yml new file mode 100644 index 00000000..56b155c4 --- /dev/null +++ b/connection/.gitlab-ci.yml @@ -0,0 +1,30 @@ +image: golang:latest + +variables: + REPO_NAME: git.icinga.com/icingadb/icingadb-connection + +before_script: + - mkdir -p $GOPATH/src/$(dirname $REPO_NAME) + - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME + - cd $GOPATH/src/$REPO_NAME + - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@git.icinga.com/".insteadOf "https://git.icinga.com/" + - go get -t ./... + +stages: + - test + - coverage + +test: + stage: test + script: + - go fmt $(go list ./... | grep -v /vendor/) + - go vet $(go list ./... | grep -v /vendor/) + - go test -race $(go list ./... | grep -v /vendor/) -cover + +coverage: + stage: coverage + script: + - ./coverage.sh + artifacts: + paths: + - coverage.html diff --git a/README.md b/connection/README.md similarity index 100% rename from README.md rename to connection/README.md diff --git a/coverage.sh b/connection/coverage.sh similarity index 100% rename from coverage.sh rename to connection/coverage.sh diff --git a/mysql.go b/connection/mysql.go similarity index 99% rename from mysql.go rename to connection/mysql.go index 3514c67b..06cbef43 100644 --- a/mysql.go +++ b/connection/mysql.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import ( "container/list" diff --git a/mysql_test.go b/connection/mysql_test.go similarity index 99% rename from mysql_test.go rename to connection/mysql_test.go index b2ca129f..fafd95e9 100644 --- a/mysql_test.go +++ b/connection/mysql_test.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import ( "context" diff --git a/mysql_utils.go b/connection/mysql_utils.go similarity index 99% rename from mysql_utils.go rename to connection/mysql_utils.go index d596e510..8220d5e9 100644 --- a/mysql_utils.go +++ b/connection/mysql_utils.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import ( "database/sql" @@ -40,7 +40,7 @@ func mkMysql(dbType string, dbDsn string) (*sql.DB, error) { mysql.SetLogger(oldlog.New(ioutil.Discard, "", 0)) - db.SetMaxOpenConns(100) + db.SetMaxOpenConns(50) db.SetMaxIdleConns(0) return db, nil diff --git a/mysql_utils_test.go b/connection/mysql_utils_test.go similarity index 98% rename from mysql_utils_test.go rename to connection/mysql_utils_test.go index b21bc171..8b5346ae 100644 --- a/mysql_utils_test.go +++ b/connection/mysql_utils_test.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import ( "errors" diff --git a/prometheus.go b/connection/prometheus.go similarity index 96% rename from prometheus.go rename to connection/prometheus.go index 8b9dc482..00094c8d 100644 --- a/prometheus.go +++ b/connection/prometheus.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import ( "github.com/prometheus/client_golang/prometheus" diff --git a/redis.go b/connection/redis.go similarity index 99% rename from redis.go rename to connection/redis.go index d71700b6..9e47b651 100644 --- a/redis.go +++ b/connection/redis.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import ( "fmt" diff --git a/redis_pipeliner.go b/connection/redis_pipeliner.go similarity index 96% rename from redis_pipeliner.go rename to connection/redis_pipeliner.go index 97e2fbbc..e58e39a3 100644 --- a/redis_pipeliner.go +++ b/connection/redis_pipeliner.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import "github.com/go-redis/redis" diff --git a/redis_pubsub.go b/connection/redis_pubsub.go similarity index 97% rename from redis_pubsub.go rename to connection/redis_pubsub.go index 1cedb303..a373ba27 100644 --- a/redis_pubsub.go +++ b/connection/redis_pubsub.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import ( "github.com/go-redis/redis" diff --git a/redis_pubsub_test.go b/connection/redis_pubsub_test.go similarity index 97% rename from redis_pubsub_test.go rename to connection/redis_pubsub_test.go index 34f6cbe9..bee59d05 100644 --- a/redis_pubsub_test.go +++ b/connection/redis_pubsub_test.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import ( "github.com/go-redis/redis" diff --git a/redis_test.go b/connection/redis_test.go similarity index 99% rename from redis_test.go rename to connection/redis_test.go index 18810aab..edc62339 100644 --- a/redis_test.go +++ b/connection/redis_test.go @@ -1,4 +1,4 @@ -package icingadb_connection +package connection import ( "github.com/go-redis/redis" diff --git a/test_db.sql b/connection/test_db.sql similarity index 100% rename from test_db.sql rename to connection/test_db.sql