upstream/icingadb
1
0
Fork 0
mirror of https://github.com/Icinga/icingadb.git synced 2026-05-28 04:35:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add test demonstrating that multiple instances may take over at the same time

Browse source 
Example output:

time="2020-12-21T09:22:29+01:00" level=info msg="Taking over." UUID=bbc39673-7281-5064-a71e-91c25dbc6643 context=HA-Testing
time="2020-12-21T09:22:29+01:00" level=info msg="Taking over." UUID=dddedcc5-507e-56d6-adb4-9c6451f6aeb7 context=HA-Testing
time="2020-12-21T09:22:29+01:00" level=info msg="Changing HA state to active"
time="2020-12-21T09:22:29+01:00" level=info msg="Changing HA state to active"
    ha_test.go:291:
        	Error Trace:	ha_test.go:291
        	Error:      	Not equal:
        	            	expected: 1
        	            	actual  : 2
        	Test:       	TestHA_ConcurrentCheckResponsibility
        	Messages:   	exactly 1 instance must be active after checkResponsibility() but 2 are active
--- FAIL: TestHA_ConcurrentCheckResponsibility (0.02s)
This commit is contained in:
Julian Brost 2020-12-21 09:31:32 +01:00
parent acc10490e9
commit b09e18825a
1 changed files with 91 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
ha/ha_test.go
View file

@ -12,6 +12,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"strconv"
"sync"
"testing"
"time"
@ -21,9 +22,7 @@ func createTestingHA(t *testing.T, redisAddr string) *HA {
redisConn := connection.NewRDBWrapper(redisAddr, "", 64)
mysqlConn, err := connection.NewDBWrapper(testbackends.MysqlTestDsn, 50)
if err != nil {
assert.Fail(t, "This test needs a working MySQL connection!")
}
require.NoError(t, err, "This test needs a working MySQL connection!")
super := supervisor.Supervisor{
ChErr: make(chan error),
@ -49,6 +48,44 @@ func createTestingHA(t *testing.T, redisAddr string) *HA {
return ha
}
func createTestingMultipleHA(t *testing.T, redisAddr string, numInstances int) ([]*HA, <-chan error) {
redisConn := connection.NewRDBWrapper(redisAddr, "", 64)
mysqlConn, err := connection.NewDBWrapper(testbackends.MysqlTestDsn, 50)
require.NoError(t, err, "This test needs a working MySQL connection!")
_, err = mysqlConn.SqlExec(mysqlTestObserver, "TRUNCATE TABLE icingadb_instance")
require.NoError(t, err, "This test needs a working MySQL connection!")
instances := make([]*HA, numInstances)
chErr := make(chan error)
for i := 0; i < numInstances; i++ {
super := supervisor.Supervisor{
ChErr: chErr,
Rdbw: redisConn,
Dbw: mysqlConn,
}
ha, _ := NewHA(&super)
hash := sha1.New()
hash.Write([]byte("derp"))
ha.super.EnvId = hash.Sum(nil)
ha.uid = uuid.NewSHA1(uuid.MustParse("551bc748-94b2-4d27-b6a4-15c52aecfe85"), []byte(strconv.Itoa(i)))
ha.logger = log.WithFields(log.Fields{
"context": "HA-Testing",
"UUID": ha.uid,
})
instances[i] = ha
}
return instances, chErr
}
var mysqlTestObserver = connection.DbIoSeconds.WithLabelValues("mysql", "test")
func TestHA_InsertInstance(t *testing.T) {
@ -202,3 +239,54 @@ func TestHA_runHA(t *testing.T) {
wg.Wait()
}
func TestHA_ConcurrentCheckResponsibility(t *testing.T) {
numAttempts := 10
numConcurrentTakeovers := 2
failed := false
for attempt := 0; !failed && attempt < numAttempts; attempt++ {
wg := sync.WaitGroup{}
wg.Add(numConcurrentTakeovers)
haInstances, chErr := createTestingMultipleHA(t, testbackends.RedisTestAddr, numConcurrentTakeovers)
for _, ha := range haInstances {
ha := ha
go func() {
defer wg.Done()
ha.checkResponsibility(&Environment{})
}()
}
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
loop:
for {
select {
case err := <-chErr:
assert.NoError(t, err, "checkResponsibility() should return no error")
if err != nil {
failed = true
}
case <-done:
break loop
}
}
numActive := 0
for _, ha := range haInstances {
if ha.state == StateActive {
numActive++
}
}
assert.Equal(t, 1, numActive, "exactly 1 instance must be active after checkResponsibility() but %d are active", numActive)
if numActive != 1 {
failed = true
}
}
}