From 417990c9a7d6f5979121e22583b6c036106ad447 Mon Sep 17 00:00:00 2001
From: Noah Hilverling <noah.hilverling@icinga.com>
Date: Wed, 30 Oct 2019 14:54:15 +0100
Subject: [PATCH] Improve SQL error handling

---
 connection/mysql.go | 58 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 49 insertions(+), 9 deletions(-)

diff --git a/connection/mysql.go b/connection/mysql.go
index 7bfe80a3..497f0b15 100644
--- a/connection/mysql.go
+++ b/connection/mysql.go
@@ -32,6 +32,31 @@ var mysqlObservers = struct {
 	DbIoSeconds.WithLabelValues("mysql", "Bulk update"),
 }
 
+var connectionErrors = []string{
+	"server has gone away",
+	"no connection to the server",
+	"Lost connection",
+	"Error while sending",
+	"is dead or not enabled",
+	"decryption failed or bad record mac",
+	"server closed the connection unexpectedly",
+	"SSL connection has been closed unexpectedly",
+	"Error writing data to the connection",
+	"Resource deadlock avoided",
+	"Transaction() on null",
+	"child connection forced to terminate due to client_idle_limit",
+	"query_wait_timeout",
+	"reset by peer",
+	"Physical connection is not usable",
+	"TCP Provider: Error code 0x68",
+	"ORA-03114",
+	"Packets out of order. Expected",
+	"Adaptive Server connection failed",
+	"Communication link failure",
+	"Deadlock found when trying to get lock",
+	"operation timed out",
+}
+
 // This is used in SqlFetchAll and SqlFetchAllQuiet
 type DbClientOrTransaction interface {
 	Query(query string, args ...interface{}) (*sql.Rows, error)
@@ -147,6 +172,21 @@ func (dbw *DBWrapper) checkConnection(isTicker bool) bool {
 	}
 }
 
+func (dbw *DBWrapper) isConnectionError(err error) bool {
+	errString := err.Error()
+	for _, str := range connectionErrors {
+		if strings.Contains(errString, str) {
+			log.WithFields(log.Fields{
+				"context": "sql",
+				"error":   errString,
+			}).Error("Got connection error. Trying again")
+			return true
+		}
+	}
+
+	return !dbw.checkConnection(false)
+}
+
 func (dbw *DBWrapper) WaitForConnection() {
 	dbw.ConnectionUpCondition.L.Lock()
 	dbw.ConnectionUpCondition.Wait()
@@ -180,7 +220,7 @@ func (dbw *DBWrapper) SqlQuery(query string, args ...interface{}) (*sql.Rows, er
 		res, err := dbw.Db.Query(query, args...)
 
 		if err != nil {
-			if !dbw.checkConnection(false) {
+			if dbw.isConnectionError(err) {
 				continue
 			}
 		}
@@ -223,7 +263,7 @@ func (dbw *DBWrapper) SqlBegin(concurrencySafety bool, quiet bool) (DbTransactio
 		}
 
 		if err != nil {
-			if !dbw.checkConnection(false) {
+			if dbw.isConnectionError(err) {
 				continue
 			}
 		}
@@ -258,7 +298,7 @@ func (dbw *DBWrapper) SqlCommit(tx DbTransaction, quiet bool) error {
 		}
 
 		if err != nil {
-			if !dbw.checkConnection(false) {
+			if dbw.isConnectionError(err) {
 				continue
 			}
 		}
@@ -293,7 +333,7 @@ func (dbw *DBWrapper) SqlRollback(tx DbTransaction, quiet bool) error {
 		}
 
 		if err != nil {
-			if !dbw.checkConnection(false) {
+			if dbw.isConnectionError(err) {
 				continue
 			}
 		}
@@ -370,7 +410,7 @@ func (dbw *DBWrapper) sqlExecInternal(db DbClientOrTransaction, opObserver prome
 		}
 
 		if err != nil {
-			if !dbw.checkConnection(false) {
+			if dbw.isConnectionError(err) {
 				continue
 			}
 		}
@@ -391,7 +431,7 @@ func (dbw *DBWrapper) sqlFetchAllInternal(db DbClientOrTransaction, queryObserve
 
 		if err != nil {
 			if _, isDb := db.(*sql.DB); isDb {
-				if !dbw.checkConnection(false) {
+				if dbw.isConnectionError(err) {
 					continue
 				}
 			}
@@ -522,7 +562,7 @@ func (dbw DBWrapper) SqlTransaction(concurrencySafety bool, retryOnConnectionFai
 				continue
 			}
 
-			if !dbw.checkConnection(false) {
+			if dbw.isConnectionError(errTx) {
 				if retryOnConnectionFailure {
 					continue
 				} else {
@@ -571,7 +611,7 @@ func (dbw *DBWrapper) SqlFetchIds(envId []byte, table string, field string) ([]s
 		)
 
 		if err != nil {
-			if !dbw.checkConnection(false) {
+			if dbw.isConnectionError(err) {
 				continue
 			}
 
@@ -611,7 +651,7 @@ func (dbw *DBWrapper) SqlFetchChecksums(table string, ids []string) (map[string]
 		rows, err := dbw.SqlQuery(query)
 
 		if err != nil {
-			if !dbw.checkConnection(false) {
+			if dbw.isConnectionError(err) {
 				continue
 			}