upstream/icingadb
1
0
Fork 0
mirror of https://github.com/Icinga/icingadb.git synced 2026-06-06 15:22:08 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Log all different failed reconnects to backends

Browse source 
E.g. the first "connection refused" and the first "hostname mismatch".

refs #351
This commit is contained in:
Alexander A. Klimov 2021-09-22 16:15:37 +02:00
parent 321db0eecf
commit f554fa9dfe
2 changed files with 16 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
pkg/config/redis.go
View file

@ -10,7 +10,6 @@ import (
"go.uber.org/zap"
"net"
"os"
"sync"
"syscall"
"time"
)
@ -45,17 +44,11 @@ func dialWithLogging(logger *zap.SugaredLogger) func(context.Context, string, st
// dial behaves like net.Dialer#DialContext, but re-tries on syscall.ECONNREFUSED.
return func(ctx context.Context, network, addr string) (conn net.Conn, err error) {
var dl net.Dialer
var logFirstError sync.Once
err = retry.WithBackoff(
ctx,
func(ctx context.Context) (err error) {
conn, err = dl.DialContext(ctx, network, addr)
logFirstError.Do(func() {
if err != nil {
logger.Warnw("Can't connect to Redis. Retrying", zap.Error(err))
}
})
return
},
func(err error) bool {
@ -66,7 +59,14 @@ func dialWithLogging(logger *zap.SugaredLogger) func(context.Context, string, st
return false
},
backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
retry.Settings{Timeout: 5 * time.Minute},
retry.Settings{
Timeout: 5 * time.Minute,
OnError: func(_ time.Duration, _ uint64, err, lastErr error) {
if lastErr == nil || err.Error() != lastErr.Error() {
logger.Warnw("Can't connect to Redis. Retrying", zap.Error(err))
}
},
},
)
err = errors.Wrap(err, "can't connect to Redis")

18
pkg/driver/driver.go
View file

@ -9,7 +9,6 @@ import (
"github.com/icinga/icingadb/pkg/retry"
"github.com/pkg/errors"
"go.uber.org/zap"
"sync"
"syscall"
"time"
)
@ -25,23 +24,22 @@ type RetryConnector struct {
// Connect implements part of the driver.Connector interface.
func (c RetryConnector) Connect(ctx context.Context) (driver.Conn, error) {
var conn driver.Conn
var logFirstError sync.Once
err := errors.Wrap(retry.WithBackoff(
ctx,
func(ctx context.Context) (err error) {
conn, err = c.Connector.Connect(ctx)
logFirstError.Do(func() {
if err != nil {
c.driver.Logger.Warnw("Can't connect to database. Retrying", zap.Error(err))
}
})
return
},
shouldRetry,
backoff.NewExponentialWithJitter(time.Millisecond*128, time.Minute*1),
retry.Settings{Timeout: timeout},
retry.Settings{
Timeout: timeout,
OnError: func(_ time.Duration, _ uint64, err, lastErr error) {
if lastErr == nil || err.Error() != lastErr.Error() {
c.driver.Logger.Warnw("Can't connect to database. Retrying", zap.Error(err))
}
},
},
), "can't connect to database")
return conn, err
}