From 5a146645f2fe20f475b4e14b2c0271ed8e99c93a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 22 Sep 2021 12:52:15 +0200 Subject: [PATCH 1/3] HA#removeInstance(): allow custom context refs #350 --- pkg/icingadb/ha.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/icingadb/ha.go b/pkg/icingadb/ha.go index c3fb8a84..bb1f8711 100644 --- a/pkg/icingadb/ha.go +++ b/pkg/icingadb/ha.go @@ -68,7 +68,7 @@ func (h *HA) Close() error { // Wait until the controller loop ended. <-h.Done() // Remove our instance from the database. - h.removeInstance() + h.removeInstance(context.Background()) // And return an error, if any. return h.Err() } @@ -259,11 +259,11 @@ func (h *HA) realize(s *icingaredisv1.IcingaStatus, t *types.UnixMilli, shouldLo return nil } -func (h *HA) removeInstance() { +func (h *HA) removeInstance(ctx context.Context) { h.logger.Debugw("Removing our row from icingadb_instance", zap.String("instance_id", hex.EncodeToString(h.instanceId))) - // Intentionally not using a context here as this is a cleanup task and h.ctx is already cancelled. + // Intentionally not using h.ctx here as it's already cancelled. query := "DELETE FROM icingadb_instance WHERE id = ?" - _, err := h.db.Exec(query, h.instanceId) + _, err := h.db.ExecContext(ctx, query, h.instanceId) if err != nil { h.logger.Warnw("Could not remove instance from database", zap.Error(err), zap.String("query", query)) } From ea7668d99a251b5df2ad12f54661bcbbdb982275 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 22 Sep 2021 12:54:19 +0200 Subject: [PATCH 2/3] HA#Close(): allow custom context refs #350 --- cmd/icingadb/main.go | 2 +- pkg/icingadb/ha.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go index daf7461b..75d4f448 100644 --- a/cmd/icingadb/main.go +++ b/cmd/icingadb/main.go @@ -69,7 +69,7 @@ func run() int { ha := icingadb.NewHA(ctx, db, heartbeat, logger) // Closing ha on exit ensures that this instance retracts its heartbeat // from the database so that another instance can take over immediately. - defer ha.Close() + defer ha.Close(context.Background()) s := icingadb.NewSync(db, rc, logger) hs := history.NewSync(db, rc, logger) rt := icingadb.NewRuntimeUpdates(db, rc, logger) diff --git a/pkg/icingadb/ha.go b/pkg/icingadb/ha.go index bb1f8711..44cd87b5 100644 --- a/pkg/icingadb/ha.go +++ b/pkg/icingadb/ha.go @@ -61,14 +61,14 @@ func NewHA(ctx context.Context, db *DB, heartbeat *icingaredis.Heartbeat, logger return ha } -// Close implements the io.Closer interface. -func (h *HA) Close() error { +// Close shuts h down. +func (h *HA) Close(ctx context.Context) error { // Cancel ctx. h.cancelCtx() // Wait until the controller loop ended. <-h.Done() // Remove our instance from the database. - h.removeInstance(context.Background()) + h.removeInstance(ctx) // And return an error, if any. return h.Err() } From 3e452d95612414d206e9e26fc9751453d86f38ca Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 22 Sep 2021 15:42:58 +0200 Subject: [PATCH 3/3] On shutdown: give up HA handover after 3s, not 5m ... not to hang for 5m if DB is down. refs #350 --- cmd/icingadb/main.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go index 75d4f448..b8503b92 100644 --- a/cmd/icingadb/main.go +++ b/cmd/icingadb/main.go @@ -19,6 +19,7 @@ import ( "os/signal" "sync" "syscall" + "time" ) const ( @@ -67,9 +68,17 @@ func run() int { heartbeat := icingaredis.NewHeartbeat(ctx, rc, logger) ha := icingadb.NewHA(ctx, db, heartbeat, logger) + // Closing ha on exit ensures that this instance retracts its heartbeat // from the database so that another instance can take over immediately. - defer ha.Close(context.Background()) + defer func() { + // Give up after 3s, not 5m (default) not to hang for 5m if DB is down. + ctx, cancelCtx := context.WithTimeout(context.Background(), 3*time.Second) + + ha.Close(ctx) + cancelCtx() + }() + s := icingadb.NewSync(db, rc, logger) hs := history.NewSync(db, rc, logger) rt := icingadb.NewRuntimeUpdates(db, rc, logger)