From 3649b367e123fe4053fd39b4a906c21198aea143 Mon Sep 17 00:00:00 2001
From: Julian Brost <julian.brost@icinga.com>
Date: Mon, 3 May 2021 13:19:18 +0200
Subject: [PATCH 1/3] Handle HA errors

---
 cmd/icingadb/main.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go
index b3927f70..d6baa096 100644
--- a/cmd/icingadb/main.go
+++ b/cmd/icingadb/main.go
@@ -95,6 +95,15 @@ func main() {
 				cancelHactx()
 			case <-hactx.Done():
 				// Nothing to do here, surrounding loop will terminate now.
+			case <-ha.Done():
+				if err := ha.Err(); err != nil {
+					panic(errors.Wrap(err, "HA exited with an error"))
+				} else if ctx.Err() == nil {
+					// ha is created as a single instance once. It should only exit if the main context is cancelled,
+					// otherwise there is no way to get Icinga DB back into a working state.
+					panic(errors.New("HA exited without an error but main context isn't cancelled"))
+				}
+				return
 			case <-ctx.Done():
 				return
 			case s := <-sig:

From e0d66fe0eeccf1b3c08f9761477375f87fdecbe7 Mon Sep 17 00:00:00 2001
From: Julian Brost <julian.brost@icinga.com>
Date: Fri, 7 May 2021 11:46:20 +0200
Subject: [PATCH 2/3] Exit with an error code if not exiting due to a signal

Icinga DB should not exit with code 0 if it does so due to an error.
---
 cmd/icingadb/main.go | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go
index d6baa096..92b249a1 100644
--- a/cmd/icingadb/main.go
+++ b/cmd/icingadb/main.go
@@ -17,7 +17,16 @@ import (
 	"syscall"
 )
 
+const (
+	ExitSuccess = 0
+	ExitFailure = 1
+)
+
 func main() {
+	os.Exit(run())
+}
+
+func run() int {
 	cmd := command.New()
 	logger := cmd.Logger
 	defer logger.Sync()
@@ -103,13 +112,13 @@ func main() {
 					// otherwise there is no way to get Icinga DB back into a working state.
 					panic(errors.New("HA exited without an error but main context isn't cancelled"))
 				}
-				return
+				return ExitFailure
 			case <-ctx.Done():
-				return
+				panic(errors.New("main context closed unexpectedly"))
 			case s := <-sig:
 				logger.Infow("Exiting due to signal", zap.String("signal", s.String()))
 				cancelCtx()
-				return
+				return ExitSuccess
 			}
 		}
 	}

From ae743817daec7eefd654ba27163c5fd7abf6a03c Mon Sep 17 00:00:00 2001
From: Julian Brost <julian.brost@icinga.com>
Date: Fri, 7 May 2021 11:42:18 +0200
Subject: [PATCH 3/3] Add comment explaining the purpose of ha.Close()

---
 cmd/icingadb/main.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmd/icingadb/main.go b/cmd/icingadb/main.go
index 92b249a1..c4645848 100644
--- a/cmd/icingadb/main.go
+++ b/cmd/icingadb/main.go
@@ -49,6 +49,8 @@ func run() int {
 	ctx, cancelCtx := context.WithCancel(context.Background())
 	heartbeat := icingaredis.NewHeartbeat(ctx, rc, logger)
 	ha := icingadb.NewHA(ctx, db, heartbeat, logger)
+	// Closing ha on exit ensures that this instance retracts its heartbeat
+	// from the database so that another instance can take over immediately.
 	defer ha.Close()
 	s := icingadb.NewSync(db, rc, logger)
 	hs := history.NewSync(db, rc, logger)