Merge pull request #17636 from roidelapluie/roidelapluie/starttime

Add start_timestamp field for unit tests

cmd/promtool/testdata/start-time-test.yml

@@ -0,0 +1,76 @@
+rule_files:
+  - rules.yml
+
+evaluation_interval: 1m
+
+tests:
+  # Test with default start_time (0 / Unix epoch).
+  - name: default_start_time
+    interval: 1m
+    promql_expr_test:
+      - expr: time()
+        eval_time: 0m
+        exp_samples:
+          - value: 0
+      - expr: time()
+        eval_time: 5m
+        exp_samples:
+          - value: 300
+
+  # Test with RFC3339 start_timestamp.
+  - name: rfc3339_start_timestamp
+    interval: 1m
+    start_timestamp: "2024-01-01T00:00:00Z"
+    promql_expr_test:
+      - expr: time()
+        eval_time: 0m
+        exp_samples:
+          - value: 1704067200
+      - expr: time()
+        eval_time: 5m
+        exp_samples:
+          - value: 1704067500
+
+  # Test with Unix timestamp start_timestamp.
+  - name: unix_timestamp_start_timestamp
+    interval: 1m
+    start_timestamp: 1609459200
+    input_series:
+      - series: test_metric
+        values: "1 1 1"
+    promql_expr_test:
+      - expr: time()
+        eval_time: 0m
+        exp_samples:
+          - value: 1609459200
+      - expr: time()
+        eval_time: 10m
+        exp_samples:
+          - value: 1609459800
+
+  # Test that input series samples are correctly timestamped with custom start_timestamp.
+  - name: samples_with_start_timestamp
+    interval: 1m
+    start_timestamp: "2024-01-01T00:00:00Z"
+    input_series:
+      - series: 'my_metric{label="test"}'
+        values: "10+10x15"
+    promql_expr_test:
+      # Query at absolute timestamp (start_timestamp = 1704067200).
+      - expr: my_metric@1704067200
+        eval_time: 5m
+        exp_samples:
+          - labels: 'my_metric{label="test"}'
+            value: 10
+      # Query at 2 minutes after start_timestamp (1704067200 + 120 = 1704067320).
+      - expr: my_metric@1704067320
+        eval_time: 5m
+        exp_samples:
+          - labels: 'my_metric{label="test"}'
+            value: 30
+      # Verify timestamp() function returns the absolute timestamp.
+      - expr: timestamp(my_metric)
+        eval_time: 5m
+        exp_samples:
+          - labels: '{label="test"}'
+            value: 1704067500

cmd/promtool/unittest.go

@@ -188,15 +188,37 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
 	return nil
 }
 
+// testStartTimestamp wraps time.Time to support custom YAML unmarshaling.
+// It can parse both RFC3339 timestamps and Unix timestamps.
+type testStartTimestamp struct {
+	time.Time
+}
+
+// UnmarshalYAML implements custom YAML unmarshaling for testStartTimestamp.
+// It accepts both RFC3339 formatted strings and numeric Unix timestamps.
+func (t *testStartTimestamp) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	parsed, err := parseTime(s)
+	if err != nil {
+		return err
+	}
+	t.Time = parsed
+	return nil
+}
+
 // testGroup is a group of input series and tests associated with it.
 type testGroup struct {
-	Interval        model.Duration   `yaml:"interval"`
-	InputSeries     []series         `yaml:"input_series"`
-	AlertRuleTests  []alertTestCase  `yaml:"alert_rule_test,omitempty"`
-	PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"`
-	ExternalLabels  labels.Labels    `yaml:"external_labels,omitempty"`
-	ExternalURL     string           `yaml:"external_url,omitempty"`
-	TestGroupName   string           `yaml:"name,omitempty"`
+	Interval        model.Duration     `yaml:"interval"`
+	InputSeries     []series           `yaml:"input_series"`
+	AlertRuleTests  []alertTestCase    `yaml:"alert_rule_test,omitempty"`
+	PromqlExprTests []promqlTestCase   `yaml:"promql_expr_test,omitempty"`
+	ExternalLabels  labels.Labels      `yaml:"external_labels,omitempty"`
+	ExternalURL     string             `yaml:"external_url,omitempty"`
+	TestGroupName   string             `yaml:"name,omitempty"`
+	StartTimestamp  testStartTimestamp `yaml:"start_timestamp,omitempty"`
 }
 
 // test performs the unit tests.
@@ -209,6 +231,8 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde
 		}()
 	}
 	// Setup testing suite.
+	// Set the start time from the test group.
+	queryOpts.StartTime = tg.StartTimestamp.Time
 	suite, err := promqltest.NewLazyLoader(tg.seriesLoadingString(), queryOpts)
 	if err != nil {
 		return []error{err}
@@ -237,7 +261,12 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde
 	groups := orderedGroups(groupsMap, groupOrderMap)
 
 	// Bounds for evaluating the rules.
-	mint := time.Unix(0, 0).UTC()
+	var mint time.Time
+	if tg.StartTimestamp.IsZero() {
+		mint = time.Unix(0, 0).UTC()
+	} else {
+		mint = tg.StartTimestamp.Time
+	}
 	maxt := mint.Add(tg.maxEvalTime())
 
 	// Optional floating point compare fuzzing.

cmd/promtool/unittest_test.go

@@ -129,6 +129,16 @@ func TestRulesUnitTest(t *testing.T) {
 			},
 			want: 0,
 		},
+		{
+			name: "Start time tests",
+			args: args{
+				files: []string{"./testdata/start-time-test.yml"},
+			},
+			queryOpts: promqltest.LazyLoaderOpts{
+				EnableAtModifier: true,
+			},
+			want: 0,
+		},
 	}
 	reuseFiles := []string{}
 	reuseCount := [2]int{}

docs/configuration/unit_testing_rules.md

@@ -48,6 +48,18 @@ input_series:
 # Name of the test group
 [ name: <string> ]
 
+# Start timestamp for the test group. This sets the base time for all samples
+# and evaluations in this test group.
+# Accepts either a Unix timestamp (e.g., 1609459200) or an RFC3339 formatted
+# timestamp (e.g., "2021-01-01T00:00:00Z").
+# Default: 0 (Unix epoch: 1970-01-01 00:00:00 UTC)
+#
+# When set:
+# - All input_series samples are timestamped starting from start_timestamp
+# - The eval_time in test cases is relative to start_timestamp
+# - The time() function returns start_timestamp + eval_time
+[ start_timestamp: <int> | <rfc3339_string> | default = 0 ]
+
 # Unit tests for the above data.
 
 # Unit tests for alerting rules. We consider the alerting rules from the input file.
@@ -137,7 +149,8 @@ values: <string>
 Prometheus allows you to have same alertname for different alerting rules. Hence in this unit testing, you have to list the union of all the firing alerts for the alertname under a single `<alert_test_case>`.
 
 ``` yaml
-# The time elapsed from time=0s when the alerts have to be checked.
+# The time elapsed from start_timestamp when the alerts have to be checked.
+# This is a duration relative to start_timestamp (which defaults to 0).
 eval_time: <duration>
 
 # Name of the alert to be tested.
@@ -168,7 +181,8 @@ exp_annotations:
 # Expression to evaluate
 expr: <string>
 
-# The time elapsed from time=0s when the expression has to be evaluated.
+# The time elapsed from start_timestamp when the expression has to be evaluated.
+# This is a duration relative to start_timestamp (which defaults to 0).
 eval_time: <duration>
 
 # Expected samples at the given evaluation time.
@@ -275,3 +289,24 @@ groups:
         summary: "Instance {{ $labels.instance }} down"
         description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
 ```
+
+### Time within tests
+
+It should be noted that in all tests, either in `alert_test_case` or
+`promql_test_case`, the output from all functions related to the current time,
+for example the `time()` and `day_of_*()` functions, will output a consistent value
+for tests.
+
+By default, at the start of the test evaluation, `time()` returns 0 (Unix epoch:
+January 1, 1970 00:00:00 UTC). The `eval_time` field specifies a duration relative
+to `start_timestamp`, so by default `time()` will return a value of `0 + eval_time`.
+
+You can configure a custom start timestamp for your tests by setting the `start_timestamp`
+field in your test group. This field accepts either:
+- A Unix timestamp (e.g., `1609459200` for January 1, 2021 00:00:00 UTC)
+- An RFC3339 formatted timestamp (e.g., `"2021-01-01T00:00:00Z"`)
+
+When you set `start_timestamp`:
+- All `input_series` samples will be timestamped starting from `start_timestamp`
+- The `eval_time` field in test cases is interpreted as a duration relative to `start_timestamp`
+- The `time()` function will return `start_timestamp + eval_time`

promql/promqltest/test.go

@@ -231,7 +231,7 @@ func raise(line int, format string, v ...any) error {
 	}
 }
 
-func parseLoad(lines []string, i int) (int, *loadCmd, error) {
+func parseLoad(lines []string, i int, startTime time.Time) (int, *loadCmd, error) {
 	if !patLoad.MatchString(lines[i]) {
 		return i, nil, raise(i, "invalid load command. (load[_with_nhcb] <step:duration>)")
 	}
@@ -245,6 +245,7 @@ func parseLoad(lines []string, i int) (int, *loadCmd, error) {
 		return i, nil, raise(i, "invalid step definition %q: %s", step, err)
 	}
 	cmd := newLoadCmd(time.Duration(gap), withNHCB)
+	cmd.startTime = startTime
 	for i+1 < len(lines) {
 		i++
 		defLine := lines[i]
@@ -579,7 +580,7 @@ func (t *test) parse(input string) error {
 		case c == "clear":
 			cmd = &clearCmd{}
 		case strings.HasPrefix(c, "load"):
-			i, cmd, err = parseLoad(lines, i)
+			i, cmd, err = parseLoad(lines, i, testStartTime)
 		case strings.HasPrefix(c, "eval"):
 			i, cmd, err = t.parseEval(lines, i)
 		default:
@@ -611,6 +612,7 @@ type loadCmd struct {
 	defs      map[uint64][]promql.Sample
 	exemplars map[uint64][]exemplar.Exemplar
 	withNHCB  bool
+	startTime time.Time
 }
 
 func newLoadCmd(gap time.Duration, withNHCB bool) *loadCmd {
@@ -620,6 +622,7 @@ func newLoadCmd(gap time.Duration, withNHCB bool) *loadCmd {
 		defs:      map[uint64][]promql.Sample{},
 		exemplars: map[uint64][]exemplar.Exemplar{},
 		withNHCB:  withNHCB,
+		startTime: testStartTime,
 	}
 }
 
@@ -632,7 +635,7 @@ func (cmd *loadCmd) set(m labels.Labels, vals ...parser.SequenceValue) {
 	h := m.Hash()
 
 	samples := make([]promql.Sample, 0, len(vals))
-	ts := testStartTime
+	ts := cmd.startTime
 	for _, v := range vals {
 		if !v.Omitted {
 			samples = append(samples, promql.Sample{
@@ -1627,6 +1630,8 @@ type LazyLoaderOpts struct {
 	// Currently defaults to false, matches the "promql-delayed-name-removal"
 	// feature flag.
 	EnableDelayedNameRemoval bool
+	// StartTime is the start time for the test. If zero, defaults to Unix epoch.
+	StartTime time.Time
 }
 
 // NewLazyLoader returns an initialized empty LazyLoader.
@@ -1652,7 +1657,12 @@ func (ll *LazyLoader) parse(input string) error {
 			continue
 		}
 		if strings.HasPrefix(strings.ToLower(patSpace.Split(l, 2)[0]), "load") {
-			_, cmd, err := parseLoad(lines, i)
+			// Determine the start time to use for loading samples.
+			startTime := testStartTime
+			if !ll.opts.StartTime.IsZero() {
+				startTime = ll.opts.StartTime
+			}
+			_, cmd, err := parseLoad(lines, i, startTime)
 			if err != nil {
 				return err
 			}