diff --git a/cmd/promtool/testdata/start-time-test.yml b/cmd/promtool/testdata/start-time-test.yml new file mode 100644 index 0000000000..b7365366f4 --- /dev/null +++ b/cmd/promtool/testdata/start-time-test.yml @@ -0,0 +1,76 @@ +rule_files: + - rules.yml + +evaluation_interval: 1m + +tests: + # Test with default start_time (0 / Unix epoch). + - name: default_start_time + interval: 1m + promql_expr_test: + - expr: time() + eval_time: 0m + exp_samples: + - value: 0 + - expr: time() + eval_time: 5m + exp_samples: + - value: 300 + + # Test with RFC3339 start_timestamp. + - name: rfc3339_start_timestamp + interval: 1m + start_timestamp: "2024-01-01T00:00:00Z" + promql_expr_test: + - expr: time() + eval_time: 0m + exp_samples: + - value: 1704067200 + - expr: time() + eval_time: 5m + exp_samples: + - value: 1704067500 + + # Test with Unix timestamp start_timestamp. + - name: unix_timestamp_start_timestamp + interval: 1m + start_timestamp: 1609459200 + input_series: + - series: test_metric + values: "1 1 1" + promql_expr_test: + - expr: time() + eval_time: 0m + exp_samples: + - value: 1609459200 + - expr: time() + eval_time: 10m + exp_samples: + - value: 1609459800 + + # Test that input series samples are correctly timestamped with custom start_timestamp. + - name: samples_with_start_timestamp + interval: 1m + start_timestamp: "2024-01-01T00:00:00Z" + input_series: + - series: 'my_metric{label="test"}' + values: "10+10x15" + promql_expr_test: + # Query at absolute timestamp (start_timestamp = 1704067200). + - expr: my_metric@1704067200 + eval_time: 5m + exp_samples: + - labels: 'my_metric{label="test"}' + value: 10 + # Query at 2 minutes after start_timestamp (1704067200 + 120 = 1704067320). + - expr: my_metric@1704067320 + eval_time: 5m + exp_samples: + - labels: 'my_metric{label="test"}' + value: 30 + # Verify timestamp() function returns the absolute timestamp. + - expr: timestamp(my_metric) + eval_time: 5m + exp_samples: + - labels: '{label="test"}' + value: 1704067500 diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 15b5171645..14557793c5 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -188,15 +188,37 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error { return nil } +// testStartTimestamp wraps time.Time to support custom YAML unmarshaling. +// It can parse both RFC3339 timestamps and Unix timestamps. +type testStartTimestamp struct { + time.Time +} + +// UnmarshalYAML implements custom YAML unmarshaling for testStartTimestamp. +// It accepts both RFC3339 formatted strings and numeric Unix timestamps. +func (t *testStartTimestamp) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + parsed, err := parseTime(s) + if err != nil { + return err + } + t.Time = parsed + return nil +} + // testGroup is a group of input series and tests associated with it. type testGroup struct { - Interval model.Duration `yaml:"interval"` - InputSeries []series `yaml:"input_series"` - AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"` - PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"` - ExternalLabels labels.Labels `yaml:"external_labels,omitempty"` - ExternalURL string `yaml:"external_url,omitempty"` - TestGroupName string `yaml:"name,omitempty"` + Interval model.Duration `yaml:"interval"` + InputSeries []series `yaml:"input_series"` + AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"` + PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"` + ExternalLabels labels.Labels `yaml:"external_labels,omitempty"` + ExternalURL string `yaml:"external_url,omitempty"` + TestGroupName string `yaml:"name,omitempty"` + StartTimestamp testStartTimestamp `yaml:"start_timestamp,omitempty"` } // test performs the unit tests. @@ -209,6 +231,8 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde }() } // Setup testing suite. + // Set the start time from the test group. + queryOpts.StartTime = tg.StartTimestamp.Time suite, err := promqltest.NewLazyLoader(tg.seriesLoadingString(), queryOpts) if err != nil { return []error{err} @@ -237,7 +261,12 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde groups := orderedGroups(groupsMap, groupOrderMap) // Bounds for evaluating the rules. - mint := time.Unix(0, 0).UTC() + var mint time.Time + if tg.StartTimestamp.IsZero() { + mint = time.Unix(0, 0).UTC() + } else { + mint = tg.StartTimestamp.Time + } maxt := mint.Add(tg.maxEvalTime()) // Optional floating point compare fuzzing. diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go index 566e0acbc6..bf4de02ccd 100644 --- a/cmd/promtool/unittest_test.go +++ b/cmd/promtool/unittest_test.go @@ -129,6 +129,16 @@ func TestRulesUnitTest(t *testing.T) { }, want: 0, }, + { + name: "Start time tests", + args: args{ + files: []string{"./testdata/start-time-test.yml"}, + }, + queryOpts: promqltest.LazyLoaderOpts{ + EnableAtModifier: true, + }, + want: 0, + }, } reuseFiles := []string{} reuseCount := [2]int{} diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index d237c8cf88..af94c414f0 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -48,6 +48,18 @@ input_series: # Name of the test group [ name: ] +# Start timestamp for the test group. This sets the base time for all samples +# and evaluations in this test group. +# Accepts either a Unix timestamp (e.g., 1609459200) or an RFC3339 formatted +# timestamp (e.g., "2021-01-01T00:00:00Z"). +# Default: 0 (Unix epoch: 1970-01-01 00:00:00 UTC) +# +# When set: +# - All input_series samples are timestamped starting from start_timestamp +# - The eval_time in test cases is relative to start_timestamp +# - The time() function returns start_timestamp + eval_time +[ start_timestamp: | | default = 0 ] + # Unit tests for the above data. # Unit tests for alerting rules. We consider the alerting rules from the input file. @@ -137,7 +149,8 @@ values: Prometheus allows you to have same alertname for different alerting rules. Hence in this unit testing, you have to list the union of all the firing alerts for the alertname under a single ``. ``` yaml -# The time elapsed from time=0s when the alerts have to be checked. +# The time elapsed from start_timestamp when the alerts have to be checked. +# This is a duration relative to start_timestamp (which defaults to 0). eval_time: # Name of the alert to be tested. @@ -168,7 +181,8 @@ exp_annotations: # Expression to evaluate expr: -# The time elapsed from time=0s when the expression has to be evaluated. +# The time elapsed from start_timestamp when the expression has to be evaluated. +# This is a duration relative to start_timestamp (which defaults to 0). eval_time: # Expected samples at the given evaluation time. @@ -275,3 +289,24 @@ groups: summary: "Instance {{ $labels.instance }} down" description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes." ``` + +### Time within tests + +It should be noted that in all tests, either in `alert_test_case` or +`promql_test_case`, the output from all functions related to the current time, +for example the `time()` and `day_of_*()` functions, will output a consistent value +for tests. + +By default, at the start of the test evaluation, `time()` returns 0 (Unix epoch: +January 1, 1970 00:00:00 UTC). The `eval_time` field specifies a duration relative +to `start_timestamp`, so by default `time()` will return a value of `0 + eval_time`. + +You can configure a custom start timestamp for your tests by setting the `start_timestamp` +field in your test group. This field accepts either: +- A Unix timestamp (e.g., `1609459200` for January 1, 2021 00:00:00 UTC) +- An RFC3339 formatted timestamp (e.g., `"2021-01-01T00:00:00Z"`) + +When you set `start_timestamp`: +- All `input_series` samples will be timestamped starting from `start_timestamp` +- The `eval_time` field in test cases is interpreted as a duration relative to `start_timestamp` +- The `time()` function will return `start_timestamp + eval_time` diff --git a/promql/promqltest/test.go b/promql/promqltest/test.go index b16433c14e..d4a11b9e50 100644 --- a/promql/promqltest/test.go +++ b/promql/promqltest/test.go @@ -231,7 +231,7 @@ func raise(line int, format string, v ...any) error { } } -func parseLoad(lines []string, i int) (int, *loadCmd, error) { +func parseLoad(lines []string, i int, startTime time.Time) (int, *loadCmd, error) { if !patLoad.MatchString(lines[i]) { return i, nil, raise(i, "invalid load command. (load[_with_nhcb] )") } @@ -245,6 +245,7 @@ func parseLoad(lines []string, i int) (int, *loadCmd, error) { return i, nil, raise(i, "invalid step definition %q: %s", step, err) } cmd := newLoadCmd(time.Duration(gap), withNHCB) + cmd.startTime = startTime for i+1 < len(lines) { i++ defLine := lines[i] @@ -579,7 +580,7 @@ func (t *test) parse(input string) error { case c == "clear": cmd = &clearCmd{} case strings.HasPrefix(c, "load"): - i, cmd, err = parseLoad(lines, i) + i, cmd, err = parseLoad(lines, i, testStartTime) case strings.HasPrefix(c, "eval"): i, cmd, err = t.parseEval(lines, i) default: @@ -611,6 +612,7 @@ type loadCmd struct { defs map[uint64][]promql.Sample exemplars map[uint64][]exemplar.Exemplar withNHCB bool + startTime time.Time } func newLoadCmd(gap time.Duration, withNHCB bool) *loadCmd { @@ -620,6 +622,7 @@ func newLoadCmd(gap time.Duration, withNHCB bool) *loadCmd { defs: map[uint64][]promql.Sample{}, exemplars: map[uint64][]exemplar.Exemplar{}, withNHCB: withNHCB, + startTime: testStartTime, } } @@ -632,7 +635,7 @@ func (cmd *loadCmd) set(m labels.Labels, vals ...parser.SequenceValue) { h := m.Hash() samples := make([]promql.Sample, 0, len(vals)) - ts := testStartTime + ts := cmd.startTime for _, v := range vals { if !v.Omitted { samples = append(samples, promql.Sample{ @@ -1627,6 +1630,8 @@ type LazyLoaderOpts struct { // Currently defaults to false, matches the "promql-delayed-name-removal" // feature flag. EnableDelayedNameRemoval bool + // StartTime is the start time for the test. If zero, defaults to Unix epoch. + StartTime time.Time } // NewLazyLoader returns an initialized empty LazyLoader. @@ -1652,7 +1657,12 @@ func (ll *LazyLoader) parse(input string) error { continue } if strings.HasPrefix(strings.ToLower(patSpace.Split(l, 2)[0]), "load") { - _, cmd, err := parseLoad(lines, i) + // Determine the start time to use for loading samples. + startTime := testStartTime + if !ll.opts.StartTime.IsZero() { + startTime = ll.opts.StartTime + } + _, cmd, err := parseLoad(lines, i, startTime) if err != nil { return err }