From 9f0b52d73a683fd10d11f554ab1d68dc30b18b2c Mon Sep 17 00:00:00 2001 From: Gabriel Filion Date: Mon, 7 Oct 2024 18:57:10 -0400 Subject: [PATCH 1/3] docs: Describe how time() is set to start at 0 in unit tests The return value of functions relating to the current time, e.g. time(), is set by promtool to start at timestamp 0 at the start of a test's evaluation. This has the very nice consequence that tests can run reliably without depending on when they are run. It does, however, mean that tests will give out results that can be unexpected by users. If this behaviour is documented, then users will be empowered to write tests for their rules that use time-dependent functions. (Closes: prometheus/docs#1464) Signed-off-by: Gabriel Filion --- docs/configuration/unit_testing_rules.md | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index d237c8cf88..13b0445c7c 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -275,3 +275,30 @@ groups: summary: "Instance {{ $labels.instance }} down" description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes." ``` + +### Time within tests + +It should be noted that in all tests, either in `alert_test_case` or +`promql_test_case`, the output from all functions related to the current time, +for example the `time()` and `day_of_*()` functions, will output a consistent value +for tests. + +At the start of the test evaluation, `time()` returns 0 and therefore when under test +`time()` will return a value of `0 + eval_time`. + +If you need to write tests for alerts that use functions relating to the current +time, make sure that the values given to your `input_series` are placed far +enough in the past, relative to the evaluation time described above. The values +can for example be negative timestamps so that with a very small `eval_time` the +alert can be expected to trigger. + +Another method that's known to work is to instead bump `eval_time` in the future +so that the timestamp output by `time()` will be a higher value and the values +in `input_series` will be far enough apart from that point in time so that the +alerts will trigger. This method has the downside of making promtool generate a +timeseries database that contains a value for each `input_series` for each +`interval` for the given test. This can become very slow relatively easily and +can end up consuming a lot of RAM for running your test. By instead using values +for `input_series` relative to the timestamp described above even though the +values go into negative numbers, you can keep `eval_time` fairly lower and avoid +making your tests run very slowly. From 1a853e23db411c93f552d46888bd8033486b860b Mon Sep 17 00:00:00 2001 From: Julien Pivotto <291750+roidelapluie@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:48:31 +0100 Subject: [PATCH 2/3] Add start_timestamp field for unit tests. This commit adds support for configuring a custom start timestamp for Prometheus unit tests, allowing tests to use realistic timestamps instead of starting at Unix epoch 0. Signed-off-by: Julien Pivotto <291750+roidelapluie@users.noreply.github.com> --- cmd/promtool/testdata/start-time-test.yml | 76 +++++++++++++++++++++++ cmd/promtool/unittest.go | 49 ++++++++++++--- cmd/promtool/unittest_test.go | 10 +++ docs/configuration/unit_testing_rules.md | 46 ++++++++------ promql/promqltest/test.go | 18 ++++-- 5 files changed, 168 insertions(+), 31 deletions(-) create mode 100644 cmd/promtool/testdata/start-time-test.yml diff --git a/cmd/promtool/testdata/start-time-test.yml b/cmd/promtool/testdata/start-time-test.yml new file mode 100644 index 0000000000..b7365366f4 --- /dev/null +++ b/cmd/promtool/testdata/start-time-test.yml @@ -0,0 +1,76 @@ +rule_files: + - rules.yml + +evaluation_interval: 1m + +tests: + # Test with default start_time (0 / Unix epoch). + - name: default_start_time + interval: 1m + promql_expr_test: + - expr: time() + eval_time: 0m + exp_samples: + - value: 0 + - expr: time() + eval_time: 5m + exp_samples: + - value: 300 + + # Test with RFC3339 start_timestamp. + - name: rfc3339_start_timestamp + interval: 1m + start_timestamp: "2024-01-01T00:00:00Z" + promql_expr_test: + - expr: time() + eval_time: 0m + exp_samples: + - value: 1704067200 + - expr: time() + eval_time: 5m + exp_samples: + - value: 1704067500 + + # Test with Unix timestamp start_timestamp. + - name: unix_timestamp_start_timestamp + interval: 1m + start_timestamp: 1609459200 + input_series: + - series: test_metric + values: "1 1 1" + promql_expr_test: + - expr: time() + eval_time: 0m + exp_samples: + - value: 1609459200 + - expr: time() + eval_time: 10m + exp_samples: + - value: 1609459800 + + # Test that input series samples are correctly timestamped with custom start_timestamp. + - name: samples_with_start_timestamp + interval: 1m + start_timestamp: "2024-01-01T00:00:00Z" + input_series: + - series: 'my_metric{label="test"}' + values: "10+10x15" + promql_expr_test: + # Query at absolute timestamp (start_timestamp = 1704067200). + - expr: my_metric@1704067200 + eval_time: 5m + exp_samples: + - labels: 'my_metric{label="test"}' + value: 10 + # Query at 2 minutes after start_timestamp (1704067200 + 120 = 1704067320). + - expr: my_metric@1704067320 + eval_time: 5m + exp_samples: + - labels: 'my_metric{label="test"}' + value: 30 + # Verify timestamp() function returns the absolute timestamp. + - expr: timestamp(my_metric) + eval_time: 5m + exp_samples: + - labels: '{label="test"}' + value: 1704067500 diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 15b5171645..75da96c2eb 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -188,15 +188,37 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error { return nil } +// testStartTimestamp wraps time.Time to support custom YAML unmarshaling. +// It can parse both RFC3339 timestamps and Unix timestamps. +type testStartTimestamp struct { + time.Time +} + +// UnmarshalYAML implements custom YAML unmarshaling for testStartTimestamp. +// It accepts both RFC3339 formatted strings and numeric Unix timestamps. +func (t *testStartTimestamp) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + parsed, err := parseTime(s) + if err != nil { + return err + } + t.Time = parsed + return nil +} + // testGroup is a group of input series and tests associated with it. type testGroup struct { - Interval model.Duration `yaml:"interval"` - InputSeries []series `yaml:"input_series"` - AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"` - PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"` - ExternalLabels labels.Labels `yaml:"external_labels,omitempty"` - ExternalURL string `yaml:"external_url,omitempty"` - TestGroupName string `yaml:"name,omitempty"` + Interval model.Duration `yaml:"interval"` + InputSeries []series `yaml:"input_series"` + AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"` + PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"` + ExternalLabels labels.Labels `yaml:"external_labels,omitempty"` + ExternalURL string `yaml:"external_url,omitempty"` + TestGroupName string `yaml:"name,omitempty"` + StartTimestamp testStartTimestamp `yaml:"start_timestamp,omitempty"` } // test performs the unit tests. @@ -209,6 +231,12 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde }() } // Setup testing suite. + // Set the start time from the test group. + if tg.StartTimestamp.IsZero() { + queryOpts.StartTime = time.Unix(0, 0).UTC() + } else { + queryOpts.StartTime = tg.StartTimestamp.Time + } suite, err := promqltest.NewLazyLoader(tg.seriesLoadingString(), queryOpts) if err != nil { return []error{err} @@ -237,7 +265,12 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde groups := orderedGroups(groupsMap, groupOrderMap) // Bounds for evaluating the rules. - mint := time.Unix(0, 0).UTC() + var mint time.Time + if tg.StartTimestamp.IsZero() { + mint = time.Unix(0, 0).UTC() + } else { + mint = tg.StartTimestamp.Time + } maxt := mint.Add(tg.maxEvalTime()) // Optional floating point compare fuzzing. diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go index 566e0acbc6..bf4de02ccd 100644 --- a/cmd/promtool/unittest_test.go +++ b/cmd/promtool/unittest_test.go @@ -129,6 +129,16 @@ func TestRulesUnitTest(t *testing.T) { }, want: 0, }, + { + name: "Start time tests", + args: args{ + files: []string{"./testdata/start-time-test.yml"}, + }, + queryOpts: promqltest.LazyLoaderOpts{ + EnableAtModifier: true, + }, + want: 0, + }, } reuseFiles := []string{} reuseCount := [2]int{} diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index 13b0445c7c..af94c414f0 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -48,6 +48,18 @@ input_series: # Name of the test group [ name: ] +# Start timestamp for the test group. This sets the base time for all samples +# and evaluations in this test group. +# Accepts either a Unix timestamp (e.g., 1609459200) or an RFC3339 formatted +# timestamp (e.g., "2021-01-01T00:00:00Z"). +# Default: 0 (Unix epoch: 1970-01-01 00:00:00 UTC) +# +# When set: +# - All input_series samples are timestamped starting from start_timestamp +# - The eval_time in test cases is relative to start_timestamp +# - The time() function returns start_timestamp + eval_time +[ start_timestamp: | | default = 0 ] + # Unit tests for the above data. # Unit tests for alerting rules. We consider the alerting rules from the input file. @@ -137,7 +149,8 @@ values: Prometheus allows you to have same alertname for different alerting rules. Hence in this unit testing, you have to list the union of all the firing alerts for the alertname under a single ``. ``` yaml -# The time elapsed from time=0s when the alerts have to be checked. +# The time elapsed from start_timestamp when the alerts have to be checked. +# This is a duration relative to start_timestamp (which defaults to 0). eval_time: # Name of the alert to be tested. @@ -168,7 +181,8 @@ exp_annotations: # Expression to evaluate expr: -# The time elapsed from time=0s when the expression has to be evaluated. +# The time elapsed from start_timestamp when the expression has to be evaluated. +# This is a duration relative to start_timestamp (which defaults to 0). eval_time: # Expected samples at the given evaluation time. @@ -283,22 +297,16 @@ It should be noted that in all tests, either in `alert_test_case` or for example the `time()` and `day_of_*()` functions, will output a consistent value for tests. -At the start of the test evaluation, `time()` returns 0 and therefore when under test -`time()` will return a value of `0 + eval_time`. +By default, at the start of the test evaluation, `time()` returns 0 (Unix epoch: +January 1, 1970 00:00:00 UTC). The `eval_time` field specifies a duration relative +to `start_timestamp`, so by default `time()` will return a value of `0 + eval_time`. -If you need to write tests for alerts that use functions relating to the current -time, make sure that the values given to your `input_series` are placed far -enough in the past, relative to the evaluation time described above. The values -can for example be negative timestamps so that with a very small `eval_time` the -alert can be expected to trigger. +You can configure a custom start timestamp for your tests by setting the `start_timestamp` +field in your test group. This field accepts either: +- A Unix timestamp (e.g., `1609459200` for January 1, 2021 00:00:00 UTC) +- An RFC3339 formatted timestamp (e.g., `"2021-01-01T00:00:00Z"`) -Another method that's known to work is to instead bump `eval_time` in the future -so that the timestamp output by `time()` will be a higher value and the values -in `input_series` will be far enough apart from that point in time so that the -alerts will trigger. This method has the downside of making promtool generate a -timeseries database that contains a value for each `input_series` for each -`interval` for the given test. This can become very slow relatively easily and -can end up consuming a lot of RAM for running your test. By instead using values -for `input_series` relative to the timestamp described above even though the -values go into negative numbers, you can keep `eval_time` fairly lower and avoid -making your tests run very slowly. +When you set `start_timestamp`: +- All `input_series` samples will be timestamped starting from `start_timestamp` +- The `eval_time` field in test cases is interpreted as a duration relative to `start_timestamp` +- The `time()` function will return `start_timestamp + eval_time` diff --git a/promql/promqltest/test.go b/promql/promqltest/test.go index b16433c14e..d4a11b9e50 100644 --- a/promql/promqltest/test.go +++ b/promql/promqltest/test.go @@ -231,7 +231,7 @@ func raise(line int, format string, v ...any) error { } } -func parseLoad(lines []string, i int) (int, *loadCmd, error) { +func parseLoad(lines []string, i int, startTime time.Time) (int, *loadCmd, error) { if !patLoad.MatchString(lines[i]) { return i, nil, raise(i, "invalid load command. (load[_with_nhcb] )") } @@ -245,6 +245,7 @@ func parseLoad(lines []string, i int) (int, *loadCmd, error) { return i, nil, raise(i, "invalid step definition %q: %s", step, err) } cmd := newLoadCmd(time.Duration(gap), withNHCB) + cmd.startTime = startTime for i+1 < len(lines) { i++ defLine := lines[i] @@ -579,7 +580,7 @@ func (t *test) parse(input string) error { case c == "clear": cmd = &clearCmd{} case strings.HasPrefix(c, "load"): - i, cmd, err = parseLoad(lines, i) + i, cmd, err = parseLoad(lines, i, testStartTime) case strings.HasPrefix(c, "eval"): i, cmd, err = t.parseEval(lines, i) default: @@ -611,6 +612,7 @@ type loadCmd struct { defs map[uint64][]promql.Sample exemplars map[uint64][]exemplar.Exemplar withNHCB bool + startTime time.Time } func newLoadCmd(gap time.Duration, withNHCB bool) *loadCmd { @@ -620,6 +622,7 @@ func newLoadCmd(gap time.Duration, withNHCB bool) *loadCmd { defs: map[uint64][]promql.Sample{}, exemplars: map[uint64][]exemplar.Exemplar{}, withNHCB: withNHCB, + startTime: testStartTime, } } @@ -632,7 +635,7 @@ func (cmd *loadCmd) set(m labels.Labels, vals ...parser.SequenceValue) { h := m.Hash() samples := make([]promql.Sample, 0, len(vals)) - ts := testStartTime + ts := cmd.startTime for _, v := range vals { if !v.Omitted { samples = append(samples, promql.Sample{ @@ -1627,6 +1630,8 @@ type LazyLoaderOpts struct { // Currently defaults to false, matches the "promql-delayed-name-removal" // feature flag. EnableDelayedNameRemoval bool + // StartTime is the start time for the test. If zero, defaults to Unix epoch. + StartTime time.Time } // NewLazyLoader returns an initialized empty LazyLoader. @@ -1652,7 +1657,12 @@ func (ll *LazyLoader) parse(input string) error { continue } if strings.HasPrefix(strings.ToLower(patSpace.Split(l, 2)[0]), "load") { - _, cmd, err := parseLoad(lines, i) + // Determine the start time to use for loading samples. + startTime := testStartTime + if !ll.opts.StartTime.IsZero() { + startTime = ll.opts.StartTime + } + _, cmd, err := parseLoad(lines, i, startTime) if err != nil { return err } From 4620c8ac71842a15fc9d74d52017c3e012b5bd80 Mon Sep 17 00:00:00 2001 From: Julien Pivotto <291750+roidelapluie@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:20:00 +0100 Subject: [PATCH 3/3] Simplify StartTime assignment in unit test setup. Remove redundant IsZero check since promqltest.LazyLoader already handles zero StartTime by defaulting to Unix epoch. Signed-off-by: Julien Pivotto <291750+roidelapluie@users.noreply.github.com> --- cmd/promtool/unittest.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 75da96c2eb..14557793c5 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -232,11 +232,7 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde } // Setup testing suite. // Set the start time from the test group. - if tg.StartTimestamp.IsZero() { - queryOpts.StartTime = time.Unix(0, 0).UTC() - } else { - queryOpts.StartTime = tg.StartTimestamp.Time - } + queryOpts.StartTime = tg.StartTimestamp.Time suite, err := promqltest.NewLazyLoader(tg.seriesLoadingString(), queryOpts) if err != nil { return []error{err}