upstream/icingadb
1
0
Fork 0
mirror of https://github.com/Icinga/icingadb.git synced 2026-06-03 14:00:11 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merge pull request #593 from Icinga/wait-for-database-to-start-rather-than-crashing-561

Browse source 
Merge network and database error retryability detection functions
This commit is contained in:
Alexander Aleksandrovič Klimov 2023-07-27 17:58:06 +02:00 committed by GitHub
commit 99de1079f8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 54 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
pkg/icingadb/db.go
View file

@ -2,9 +2,7 @@ package icingadb
import (
"context"
sqlDriver "database/sql/driver"
"fmt"
"github.com/go-sql-driver/mysql"
"github.com/icinga/icingadb/internal"
"github.com/icinga/icingadb/pkg/backoff"
"github.com/icinga/icingadb/pkg/com"
@ -15,7 +13,6 @@ import (
"github.com/icinga/icingadb/pkg/retry"
"github.com/icinga/icingadb/pkg/utils"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
@ -339,7 +336,7 @@ func (db *DB) BulkExec(
return nil
},
IsRetryable,
retry.Retryable,
backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
retry.Settings{},
)
@ -404,7 +401,7 @@ func (db *DB) NamedBulkExec(
return nil
},
IsRetryable,
retry.Retryable,
backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
retry.Settings{},
)
@ -477,7 +474,7 @@ func (db *DB) NamedBulkExecTx(
return nil
},
IsRetryable,
retry.Retryable,
backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
retry.Settings{},
)
@ -674,57 +671,3 @@ func (db *DB) log(ctx context.Context, query string, counter *com.Counter) perio
db.logger.Debugf("Finished executing %q with %d rows in %s", query, counter.Total(), tick.Elapsed)
}))
}
// IsRetryable checks whether the given error is retryable.
func IsRetryable(err error) bool {
if errors.Is(err, sqlDriver.ErrBadConn) {
return true
}
if errors.Is(err, mysql.ErrInvalidConn) {
return true
}
var e *mysql.MySQLError
if errors.As(err, &e) {
switch e.Number {
case 1053, 1205, 1213, 2006:
// 1053: Server shutdown in progress
// 1205: Lock wait timeout
// 1213: Deadlock found when trying to get lock
// 2006: MySQL server has gone away
return true
default:
return false
}
}
var pe *pq.Error
if errors.As(err, &pe) {
switch pe.Code {
case "08000", // connection_exception
"08006", // connection_failure
"08001", // sqlclient_unable_to_establish_sqlconnection
"08004", // sqlserver_rejected_establishment_of_sqlconnection
"40001", // serialization_failure
"40P01", // deadlock_detected
"54000", // program_limit_exceeded
"55006", // object_in_use
"55P03", // lock_not_available
"57P01", // admin_shutdown
"57P02", // crash_shutdown
"57P03", // cannot_connect_now
"58000", // system_error
"58030", // io_error
"XX000": // internal_error
return true
default:
if strings.HasPrefix(string(pe.Code), "53") {
// Class 53 - Insufficient Resources
return true
}
}
}
return false
}

2
pkg/icingadb/ha.go
View file

@ -319,7 +319,7 @@ func (h *HA) realize(ctx context.Context, s *icingaredisv1.IcingaStatus, t *type
return nil
},
IsRetryable,
retry.Retryable,
backoff.NewExponentialWithJitter(time.Millisecond*256, time.Second*3),
retry.Settings{
OnError: func(_ time.Duration, attempt uint64, err, lastErr error) {

50
pkg/retry/retry.go
View file

@ -2,9 +2,13 @@ package retry
import (
"context"
"database/sql/driver"
"github.com/go-sql-driver/mysql"
"github.com/icinga/icingadb/pkg/backoff"
"github.com/lib/pq"
"github.com/pkg/errors"
"net"
"strings"
"syscall"
"time"
)
@ -130,5 +134,51 @@ func Retryable(err error) bool {
return true
}
if errors.Is(err, driver.ErrBadConn) {
return true
}
if errors.Is(err, mysql.ErrInvalidConn) {
return true
}
var e *mysql.MySQLError
if errors.As(err, &e) {
switch e.Number {
case 1053, 1205, 1213, 2006:
// 1053: Server shutdown in progress
// 1205: Lock wait timeout
// 1213: Deadlock found when trying to get lock
// 2006: MySQL server has gone away
return true
default:
return false
}
}
var pe *pq.Error
if errors.As(err, &pe) {
switch pe.Code {
case "08000", // connection_exception
"08006", // connection_failure
"08001", // sqlclient_unable_to_establish_sqlconnection
"08004", // sqlserver_rejected_establishment_of_sqlconnection
"40001", // serialization_failure
"40P01", // deadlock_detected
"54000", // program_limit_exceeded
"55006", // object_in_use
"55P03", // lock_not_available
"57P01", // admin_shutdown
"57P02", // crash_shutdown
"57P03", // cannot_connect_now
"58000", // system_error
"58030", // io_error
"XX000": // internal_error
return true
default:
// Class 53 - Insufficient Resources
return strings.HasPrefix(string(pe.Code), "53")
}
}
return false
}