From 64a106c5dd31b59418192e0b98f47ff0d99713db Mon Sep 17 00:00:00 2001 From: Arthur Silva Sens Date: Mon, 7 Dec 2020 18:29:43 -0300 Subject: [PATCH] Logging added for when compaction takes more than the block time range (#8151) * Logging added for when compaction takes more than the block time range Signed-off-by: arthursens * Log only if no errors were already logged Signed-off-by: arthursens * Log duration as human readable string Signed-off-by: arthursens * Move logging from compactHead() to Compact() Signed-off-by: arthursens * Compute duration of all head compactions plus wal truncation Signed-off-by: arthursens * Remove named return added os first commits Signed-off-by: arthursens * Address nits Signed-off-by: arthursens * Change miliseconds to seconds to make fuzzit tests happy Signed-off-by: ArthurSens --- tsdb/db.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tsdb/db.go b/tsdb/db.go index f63437fc6a..dc94806dff 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -805,6 +805,7 @@ func (db *DB) Compact() (returnErr error) { ).Err() }() + start := time.Now() // Check whether we have pending head blocks that are ready to be persisted. // They have the highest priority. for { @@ -839,6 +840,15 @@ func (db *DB) Compact() (returnErr error) { return errors.Wrap(err, "WAL truncation in Compact") } + compactionDuration := time.Since(start) + // TODO: change to milliseconds once fuzzit tests are removed + if int64(compactionDuration.Seconds())*1000 > db.head.chunkRange.Load() { + level.Warn(db.logger).Log( + "msg", "Head compaction took longer than the block time range, compactions are falling behind and won't be able to catch up", + "duration", compactionDuration.String(), + "block_range", db.head.chunkRange.Load(), + ) + } return db.compactBlocks() }