upstream/icingadb
1
0
Fork 0
mirror of https://github.com/Icinga/icingadb.git synced 2026-05-28 04:35:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Reduce "Can't update or insert instance. Retrying" log noise

Browse source 
Only log after the third retry with the info level. Also, before
executing the transaction, sleep dependent on the retry count.
This commit is contained in:
Eric Lippmann 2021-05-25 16:35:41 +02:00
parent 372f5cae7c
commit 9ba2c01ce2
1 changed files with 10 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
pkg/icingadb/ha.go
View file

@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/hex"
"github.com/google/uuid"
"github.com/icinga/icingadb/pkg/backoff"
v1 "github.com/icinga/icingadb/pkg/icingadb/v1"
"github.com/icinga/icingadb/pkg/icingaredis"
icingaredisv1 "github.com/icinga/icingadb/pkg/icingaredis/v1"
@ -159,11 +160,10 @@ func (h *HA) controller() {
}
func (h *HA) realize(s *icingaredisv1.IcingaStatus, t *types.UnixMilli, shouldLog bool) error {
// boff := backoff.NewExponentialWithJitter(time.Millisecond*1, time.Second*1)
boff := backoff.NewExponentialWithJitter(time.Millisecond*256, time.Second*3)
for attempt := 0; true; attempt++ {
// sleep := boff(uint64(attempt))
// h.logger.Debugf("Sleeping for %s..", sleep)
// time.Sleep(sleep)
sleep := boff(uint64(attempt))
time.Sleep(sleep)
ctx, cancel := context.WithCancel(h.ctx)
tx, err := h.db.BeginTxx(ctx, &sql.TxOptions{
@ -223,7 +223,12 @@ func (h *HA) realize(s *icingaredisv1.IcingaStatus, t *types.UnixMilli, shouldLo
h.logger.Errorw("Can't update or insert instance", zap.Error(err))
break
} else {
h.logger.Infow("Can't update or insert instance. Retrying", zap.Error(err))
if attempt > 2 {
// Log with info level after third attempt
h.logger.Infow("Can't update or insert instance. Retrying", zap.Error(err), zap.Int("retry count", attempt))
} else {
h.logger.Debugw("Can't update or insert instance. Retrying", zap.Error(err), zap.Int("retry count", attempt))
}
continue
}
}