upstream/icingadb
1
0
Fork 0
mirror of https://github.com/Icinga/icingadb.git synced 2026-06-03 14:00:11 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merge network and database error retryability detection functions

Browse source 
so that connection attempts will also be re-tried on RDBMS-specific errors,
e.g. Postgres' 57P03 (the database system is starting up), not to crash.
On the other hand, SQL operations which are safe to retry on SQL errors
are also safe to retry on network errors.
This commit is contained in:
Alexander A. Klimov 2023-05-26 12:21:09 +02:00
parent 4ed4db3ab6
commit e776c99ede
3 changed files with 54 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
pkg/icingadb/db.go
View file

@ -2,9 +2,7 @@ package icingadb
import (
"context"
sqlDriver "database/sql/driver"
"fmt"
"github.com/go-sql-driver/mysql"
"github.com/icinga/icingadb/internal"
"github.com/icinga/icingadb/pkg/backoff"
"github.com/icinga/icingadb/pkg/com"
@ -15,7 +13,6 @@ import (
"github.com/icinga/icingadb/pkg/retry"
"github.com/icinga/icingadb/pkg/utils"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
@ -339,7 +336,7 @@ func (db *DB) BulkExec(
return nil
},
IsRetryable,
retry.Retryable,
backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
retry.Settings{},
)
@ -404,7 +401,7 @@ func (db *DB) NamedBulkExec(
return nil
},
IsRetryable,
retry.Retryable,
backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
retry.Settings{},
)
@ -477,7 +474,7 @@ func (db *DB) NamedBulkExecTx(
return nil
},
IsRetryable,
retry.Retryable,
backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
retry.Settings{},
)
@ -674,57 +671,3 @@ func (db *DB) log(ctx context.Context, query string, counter *com.Counter) perio
db.logger.Debugf("Finished executing %q with %d rows in %s", query, counter.Total(), tick.Elapsed)
}))
}
// IsRetryable checks whether the given error is retryable.
func IsRetryable(err error) bool {
if errors.Is(err, sqlDriver.ErrBadConn) {
return true
}
if errors.Is(err, mysql.ErrInvalidConn) {
return true
}
var e *mysql.MySQLError
if errors.As(err, &e) {
switch e.Number {
case 1053, 1205, 1213, 2006:
// 1053: Server shutdown in progress
// 1205: Lock wait timeout
// 1213: Deadlock found when trying to get lock
// 2006: MySQL server has gone away
return true
default:
return false
}
}
var pe *pq.Error
if errors.As(err, &pe) {
switch pe.Code {
case "08000", // connection_exception
"08006", // connection_failure
"08001", // sqlclient_unable_to_establish_sqlconnection
"08004", // sqlserver_rejected_establishment_of_sqlconnection
"40001", // serialization_failure
"40P01", // deadlock_detected
"54000", // program_limit_exceeded
"55006", // object_in_use
"55P03", // lock_not_available
"57P01", // admin_shutdown
"57P02", // crash_shutdown
"57P03", // cannot_connect_now
"58000", // system_error
"58030", // io_error
"XX000": // internal_error
return true
default:
if strings.HasPrefix(string(pe.Code), "53") {
// Class 53 - Insufficient Resources
return true
}
}
}
return false
}

2
pkg/icingadb/ha.go
View file

@ -319,7 +319,7 @@ func (h *HA) realize(ctx context.Context, s *icingaredisv1.IcingaStatus, t *type
return nil
},
IsRetryable,
retry.Retryable,
backoff.NewExponentialWithJitter(time.Millisecond*256, time.Second*3),
retry.Settings{
OnError: func(_ time.Duration, attempt uint64, err, lastErr error) {

50
pkg/retry/retry.go
View file

@ -2,9 +2,13 @@ package retry
import (
"context"
"database/sql/driver"
"github.com/go-sql-driver/mysql"
"github.com/icinga/icingadb/pkg/backoff"
"github.com/lib/pq"
"github.com/pkg/errors"
"net"
"strings"
"syscall"
"time"
)
@ -130,5 +134,51 @@ func Retryable(err error) bool {
return true
}
if errors.Is(err, driver.ErrBadConn) {
return true
}
if errors.Is(err, mysql.ErrInvalidConn) {
return true
}
var e *mysql.MySQLError
if errors.As(err, &e) {
switch e.Number {
case 1053, 1205, 1213, 2006:
// 1053: Server shutdown in progress
// 1205: Lock wait timeout
// 1213: Deadlock found when trying to get lock
// 2006: MySQL server has gone away
return true
default:
return false
}
}
var pe *pq.Error
if errors.As(err, &pe) {
switch pe.Code {
case "08000", // connection_exception
"08006", // connection_failure
"08001", // sqlclient_unable_to_establish_sqlconnection
"08004", // sqlserver_rejected_establishment_of_sqlconnection
"40001", // serialization_failure
"40P01", // deadlock_detected
"54000", // program_limit_exceeded
"55006", // object_in_use
"55P03", // lock_not_available
"57P01", // admin_shutdown
"57P02", // crash_shutdown
"57P03", // cannot_connect_now
"58000", // system_error
"58030", // io_error
"XX000": // internal_error
return true
default:
// Class 53 - Insufficient Resources
return strings.HasPrefix(string(pe.Code), "53")
}
}
return false
}