upstream/icingadb
1
0
Fork 0
mirror of https://github.com/Icinga/icingadb.git synced 2026-06-08 16:34:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merge branch 'feature/improve-sql-error-handling'

Browse source
This commit is contained in:
Alexander A. Klimov 2019-11-04 14:26:33 +01:00
commit f92b2ecee2
1 changed files with 49 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
connection/mysql.go
View file

@ -34,6 +34,31 @@ var mysqlObservers = struct {
DbIoSeconds.WithLabelValues("mysql", "Bulk update"),
}
var connectionErrors = []string{
"server has gone away",
"no connection to the server",
"Lost connection",
"Error while sending",
"is dead or not enabled",
"decryption failed or bad record mac",
"server closed the connection unexpectedly",
"SSL connection has been closed unexpectedly",
"Error writing data to the connection",
"Resource deadlock avoided",
"Transaction() on null",
"child connection forced to terminate due to client_idle_limit",
"query_wait_timeout",
"reset by peer",
"Physical connection is not usable",
"TCP Provider: Error code 0x68",
"ORA-03114",
"Packets out of order. Expected",
"Adaptive Server connection failed",
"Communication link failure",
"Deadlock found when trying to get lock",
"operation timed out",
}
// This is used in SqlFetchAll and SqlFetchAllQuiet
type DbClientOrTransaction interface {
Query(query string, args ...interface{}) (*sql.Rows, error)
@ -149,6 +174,21 @@ func (dbw *DBWrapper) checkConnection(isTicker bool) bool {
}
}
func (dbw *DBWrapper) isConnectionError(err error) bool {
errString := err.Error()
for _, str := range connectionErrors {
if strings.Contains(errString, str) {
log.WithFields(log.Fields{
"context": "sql",
"error": errString,
}).Error("Got connection error. Trying again")
return true
}
}
return !dbw.checkConnection(false)
}
func (dbw *DBWrapper) WaitForConnection() {
dbw.ConnectionUpCondition.L.Lock()
dbw.ConnectionUpCondition.Wait()
@ -182,7 +222,7 @@ func (dbw *DBWrapper) SqlQuery(query string, args ...interface{}) (*sql.Rows, er
res, err := dbw.Db.Query(query, args...)
if err != nil {
if !dbw.checkConnection(false) {
if dbw.isConnectionError(err) {
continue
}
}
@ -225,7 +265,7 @@ func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (DbTransactio
}
if err != nil {
if !dbw.checkConnection(false) {
if dbw.isConnectionError(err) {
continue
}
}
@ -260,7 +300,7 @@ func (dbw *DBWrapper) SqlCommit(tx DbTransaction, quiet bool) error {
}
if err != nil {
if !dbw.checkConnection(false) {
if dbw.isConnectionError(err) {
continue
}
}
@ -295,7 +335,7 @@ func (dbw *DBWrapper) SqlRollback(tx DbTransaction, quiet bool) error {
}
if err != nil {
if !dbw.checkConnection(false) {
if dbw.isConnectionError(err) {
continue
}
}
@ -372,7 +412,7 @@ func (dbw *DBWrapper) sqlExecInternal(db DbClientOrTransaction, opObserver prome
}
if err != nil {
if !dbw.checkConnection(false) {
if dbw.isConnectionError(err) {
continue
}
}
@ -393,7 +433,7 @@ func (dbw *DBWrapper) sqlFetchAllInternal(db DbClientOrTransaction, queryObserve
if err != nil {
if _, isDb := db.(*sql.DB); isDb {
if !dbw.checkConnection(false) {
if dbw.isConnectionError(err) {
continue
}
}
@ -524,7 +564,7 @@ func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFai
continue
}
if !dbw.checkConnection(false) {
if dbw.isConnectionError(errTx) {
if retryOnConnectionFailure {
continue
} else {
@ -573,7 +613,7 @@ func (dbw *DBWrapper) SqlFetchIds(envId []byte, table string, field string) ([]s
)
if err != nil {
if !dbw.checkConnection(false) {
if dbw.isConnectionError(err) {
continue
}
@ -613,7 +653,7 @@ func (dbw *DBWrapper) SqlFetchChecksums(table string, ids []string) (map[string]
rows, err := dbw.SqlQuery(query)
if err != nil {
if !dbw.checkConnection(false) {
if dbw.isConnectionError(err) {
continue
}