diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go index daf7461b..b8503b92 100644 --- a/cmd/icingadb/main.go +++ b/cmd/icingadb/main.go @@ -19,6 +19,7 @@ import ( "os/signal" "sync" "syscall" + "time" ) const ( @@ -67,9 +68,17 @@ func run() int { heartbeat := icingaredis.NewHeartbeat(ctx, rc, logger) ha := icingadb.NewHA(ctx, db, heartbeat, logger) + // Closing ha on exit ensures that this instance retracts its heartbeat // from the database so that another instance can take over immediately. - defer ha.Close() + defer func() { + // Give up after 3s, not 5m (default) not to hang for 5m if DB is down. + ctx, cancelCtx := context.WithTimeout(context.Background(), 3*time.Second) + + ha.Close(ctx) + cancelCtx() + }() + s := icingadb.NewSync(db, rc, logger) hs := history.NewSync(db, rc, logger) rt := icingadb.NewRuntimeUpdates(db, rc, logger) diff --git a/pkg/icingadb/ha.go b/pkg/icingadb/ha.go index c3fb8a84..44cd87b5 100644 --- a/pkg/icingadb/ha.go +++ b/pkg/icingadb/ha.go @@ -61,14 +61,14 @@ func NewHA(ctx context.Context, db *DB, heartbeat *icingaredis.Heartbeat, logger return ha } -// Close implements the io.Closer interface. -func (h *HA) Close() error { +// Close shuts h down. +func (h *HA) Close(ctx context.Context) error { // Cancel ctx. h.cancelCtx() // Wait until the controller loop ended. <-h.Done() // Remove our instance from the database. - h.removeInstance() + h.removeInstance(ctx) // And return an error, if any. return h.Err() } @@ -259,11 +259,11 @@ func (h *HA) realize(s *icingaredisv1.IcingaStatus, t *types.UnixMilli, shouldLo return nil } -func (h *HA) removeInstance() { +func (h *HA) removeInstance(ctx context.Context) { h.logger.Debugw("Removing our row from icingadb_instance", zap.String("instance_id", hex.EncodeToString(h.instanceId))) - // Intentionally not using a context here as this is a cleanup task and h.ctx is already cancelled. + // Intentionally not using h.ctx here as it's already cancelled. query := "DELETE FROM icingadb_instance WHERE id = ?" - _, err := h.db.Exec(query, h.instanceId) + _, err := h.db.ExecContext(ctx, query, h.instanceId) if err != nil { h.logger.Warnw("Could not remove instance from database", zap.Error(err), zap.String("query", query)) }