From 69906bb4f5f9e62255bced373c56fc13a3f61093 Mon Sep 17 00:00:00 2001 From: Kapil Lamba Date: Thu, 19 Jun 2025 03:27:39 +0530 Subject: [PATCH] Add script for converting PromQL tests to new syntax format (#16562) Signed-off-by: Kapil Lamba Co-authored-by: Neeraj Gartia <80708727+NeerajGartia21@users.noreply.github.com> --- promql/promqltest/README.md | 27 ++ promql/promqltest/cmd/migrate/main.go | 33 +++ promql/promqltest/test_migrate.go | 200 +++++++++++++ promql/promqltest/test_migrate_test.go | 381 +++++++++++++++++++++++++ 4 files changed, 641 insertions(+) create mode 100644 promql/promqltest/cmd/migrate/main.go create mode 100644 promql/promqltest/test_migrate.go create mode 100644 promql/promqltest/test_migrate_test.go diff --git a/promql/promqltest/README.md b/promql/promqltest/README.md index 5ac0d02adb..84a0e69f3a 100644 --- a/promql/promqltest/README.md +++ b/promql/promqltest/README.md @@ -164,3 +164,30 @@ There can be multiple `` lines for a given ``. Each `` valid Every `` line must match at least one corresponding annotation or error. If at least one `` line of type `warn` or `info` is present, then all corresponding annotations must have a matching `expect` line. + +#### Migrating Test Files to the New Syntax + +- All `.test` files in the directory specified by the --dir flag will be updated in place. +- Deprecated syntax will be replaced with the recommended `expect` line statements. + +Usage: +```sh +go run ./promql/promqltest/cmd/migrate/main.go --mode=strict [--dir=] +``` + +The `--mode` flag controls how expectations are migrated: +- `strict`: Strictly migrates all expectations to the new syntax. + This is probably more verbose than intended because the old syntax + implied many constraints that are often not needed. +- `basic`: Like `strict` but never creates `no_info` and `no_warn` + expectations. This can be a good starting point to manually add + `no_info` and `no_warn` expectations and/or remove `info` and + `warn` expectations as needed. +- `tolerant`: Only creates `expect fail` and `expect ordered` where + appropriate. All desired expectations about presence or absence + of `info` and `warn` have to be added manually. + +All three modes create valid passing tests from previously passing tests. +`basic` and `tolerant` just test fewer expectations than the previous tests. + +The --dir flag specifies the directory containing test files to migrate. diff --git a/promql/promqltest/cmd/migrate/main.go b/promql/promqltest/cmd/migrate/main.go new file mode 100644 index 0000000000..a506f084c5 --- /dev/null +++ b/promql/promqltest/cmd/migrate/main.go @@ -0,0 +1,33 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/prometheus/prometheus/promql/promqltest" +) + +func main() { + mode := flag.String("mode", "strict", "Migration mode: strict, basic, or tolerant") + dir := flag.String("dir", "", "Directory to migrate") + flag.Parse() + + if err := promqltest.MigrateTestData(*mode, *dir); err != nil { + fmt.Printf("Error migrating test files: %v\n", err) + os.Exit(1) + } +} diff --git a/promql/promqltest/test_migrate.go b/promql/promqltest/test_migrate.go new file mode 100644 index 0000000000..0b233e7592 --- /dev/null +++ b/promql/promqltest/test_migrate.go @@ -0,0 +1,200 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promqltest + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/grafana/regexp" +) + +const defaultTestDataDir = "promql/promqltest/testdata" + +var ( + evalRegex = regexp.MustCompile(`^(eval |eval_fail |eval_warn |eval_info |eval_ordered )(.*)$`) + indentRegex = regexp.MustCompile(`^([ \t]+)\S`) +) + +type MigrateMode int + +const ( + MigrateStrict MigrateMode = iota + MigrateBasic + MigrateTolerant +) + +func ParseMigrateMode(s string) (MigrateMode, error) { + switch s { + case "strict": + return MigrateStrict, nil + case "basic": + return MigrateBasic, nil + case "tolerant": + return MigrateTolerant, nil + default: + return MigrateStrict, fmt.Errorf("invalid mode: %s", s) + } +} + +// MigrateTestData migrates all PromQL test files to the new syntax format. +// It applies annotation rules based on the provided migration mode ("strict", "basic", or "tolerant"). +// The function parses each .test file, converts it to the new syntax and overwrites the file. +func MigrateTestData(mode, dir string) error { + if dir == "" { + dir = defaultTestDataDir + } + + migrationMode, err := ParseMigrateMode(mode) + if err != nil { + return fmt.Errorf("failed to parse mode: %w", err) + } + + files, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("failed to read testdata directory: %w", err) + } + + annotationMap := map[MigrateMode]map[string][]string{ + MigrateStrict: { + "eval_fail": {"expect fail", "expect no_warn", "expect no_info"}, + "eval_warn": {"expect warn", "expect no_info"}, + "eval_info": {"expect info", "expect no_warn"}, + "eval_ordered": {"expect ordered", "expect no_warn", "expect no_info"}, + "eval": {"expect no_warn", "expect no_info"}, + }, + MigrateBasic: { + "eval_fail": {"expect fail"}, + "eval_warn": {"expect warn"}, + "eval_info": {"expect info"}, + "eval_ordered": {"expect ordered"}, + }, + MigrateTolerant: { + "eval_fail": {"expect fail"}, + "eval_ordered": {"expect ordered"}, + }, + } + + for _, file := range files { + if file.IsDir() || !strings.HasSuffix(file.Name(), ".test") { + continue + } + + path := filepath.Join(dir, file.Name()) + content, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", path, err) + } + + lines := strings.Split(string(content), "\n") + processedLines, err := processTestFileLines(lines, annotationMap[migrationMode], evalRegex) + if err != nil { + return fmt.Errorf("error processing file %s: %w", path, err) + } + + if err := os.WriteFile(path, []byte(strings.Join(processedLines, "\n")), 0o644); err != nil { + return fmt.Errorf("failed to write file %s: %w", path, err) + } + } + return nil +} + +func processTestFileLines( + lines []string, + annotationMap map[string][]string, + evalRegex *regexp.Regexp, +) (result []string, err error) { + for i := 0; i < len(lines); i++ { + startLine := lines[i] + matches := evalRegex.FindStringSubmatch(strings.TrimSpace(startLine)) + if matches == nil { + result = append(result, startLine) + continue + } + + var inputBlock []string + var outputBlock []string + skipBlock := false + i++ + for i < len(lines) { + inputBlock = append(inputBlock, lines[i]) + if strings.HasPrefix(strings.TrimSpace(lines[i]), "expect ") { + skipBlock = true + } + if i+1 < len(lines) && evalRegex.MatchString(strings.TrimSpace(lines[i+1])) { + break + } + i++ + } + + if skipBlock { + result = append(result, startLine) + result = append(result, inputBlock...) + continue + } + + // Get leading whitespace from startLine using indentRegex. + leadingWS := "" + if indentMatch := indentRegex.FindStringSubmatch(startLine); indentMatch != nil { + leadingWS = indentMatch[1] + } + + command := strings.TrimSpace(matches[1]) + expression := matches[2] + var annotations []string + result = append(result, leadingWS+fmt.Sprintf("eval %s", expression)) + + // Detecting indentation style (tab or space) from the first non-empty, indented line. + indent := " " + for _, line := range inputBlock { + if indentMatch := indentRegex.FindStringSubmatch(line); indentMatch != nil { + indent = indentMatch[1] + break + } + } + + for _, annotation := range annotationMap[command] { + annotations = append(annotations, indent+annotation) + } + + for _, line := range inputBlock { + trimmedLine := strings.TrimSpace(line) + switch { + case strings.HasPrefix(trimmedLine, "expected_fail_message"): + msg := strings.TrimPrefix(trimmedLine, "expected_fail_message ") + for j, s := range annotations { + if strings.Contains(s, "expect fail") { + annotations[j] = indent + fmt.Sprintf("expect fail msg:%s", msg) + } + } + case strings.HasPrefix(trimmedLine, "expected_fail_regexp"): + regex := strings.TrimPrefix(trimmedLine, "expected_fail_regexp ") + for j, s := range annotations { + if strings.Contains(s, "expect fail") { + annotations[j] = indent + fmt.Sprintf("expect fail regex:%s", regex) + } + } + default: + outputBlock = append(outputBlock, line) + } + } + + result = append(result, annotations...) + result = append(result, outputBlock...) + } + + return result, nil +} diff --git a/promql/promqltest/test_migrate_test.go b/promql/promqltest/test_migrate_test.go new file mode 100644 index 0000000000..fcf7e9db03 --- /dev/null +++ b/promql/promqltest/test_migrate_test.go @@ -0,0 +1,381 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promqltest + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeTestFile(t *testing.T, dir, content string) string { + t.Helper() + testFile := filepath.Join(dir, "testcase.test") + require.NoError(t, os.WriteFile(testFile, []byte(content), 0o644)) + return testFile +} + +func readTestFile(t *testing.T, path string) string { + t.Helper() + output, err := os.ReadFile(path) + require.NoError(t, err) + return string(output) +} + +func assertMigration(t *testing.T, mode, input, expected string) { + dir := t.TempDir() + testFile := writeTestFile(t, dir, input) + + err := MigrateTestData(mode, dir) + require.NoError(t, err) + + output := readTestFile(t, testFile) + require.Equal(t, expected, output) +} + +func TestMigrateTestData_BasicMode(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "Basic mode with fail", + input: ` + eval_fail instant at 1m sum(foo) + expected_fail_message something went wrong + {src="a"} 1 + `, + expected: ` + eval instant at 1m sum(foo) + expect fail msg:something went wrong + {src="a"} 1 + `, + }, + { + name: "Basic mode with warn", + input: ` + eval_warn instant at 2m avg(bar) + {src="a"} 1 + `, + expected: ` + eval instant at 2m avg(bar) + expect warn + {src="a"} 1 + `, + }, + { + name: "Basic mode with info", + input: ` + eval_info instant at 3m min(baz) + {src="a"} 1 + `, + expected: ` + eval instant at 3m min(baz) + expect info + {src="a"} 1 + `, + }, + { + name: "Basic mode with ordered", + input: ` + eval_ordered instant at 4m max(qux) + {src="a"} 1 + `, + expected: ` + eval instant at 4m max(qux) + expect ordered + {src="a"} 1 + `, + }, + { + name: "Basic mode with multiple eval blocks", + input: ` + eval_fail instant at 1m sum(foo) + expected_fail_message something else went wrong + {src="a"} 1 + + eval_warn instant at 2m avg(bar) + {src="a"} 1 + `, + expected: ` + eval instant at 1m sum(foo) + expect fail msg:something else went wrong + {src="a"} 1 + + eval instant at 2m avg(bar) + expect warn + {src="a"} 1 + `, + }, + { + name: "Basic mode with already migrated syntax (no changes)", + input: ` + eval instant at 1m sum(foo) + expect fail msg:something went wrong + {src="a"} 1 + `, + expected: ` + eval instant at 1m sum(foo) + expect fail msg:something went wrong + {src="a"} 1 + `, + }, + { + name: "Basic mode with only comments and whitespace", + input: ` + # This is a comment + + `, + expected: ` + # This is a comment + + `, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assertMigration(t, "basic", tc.input, tc.expected) + }) + } +} + +func TestMigrateTestData_StrictMode(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "Strict mode with fail", + input: ` + eval_fail instant at 1m sum(foo) + expected_fail_message something went wrong + {src="a"} 1 +`, + expected: ` + eval instant at 1m sum(foo) + expect fail msg:something went wrong + expect no_warn + expect no_info + {src="a"} 1 +`, + }, + { + name: "Strict mode with warn", + input: ` + eval_warn instant at 2m avg(bar) + {src="a"} 1 +`, + expected: ` + eval instant at 2m avg(bar) + expect warn + expect no_info + {src="a"} 1 +`, + }, + { + name: "Strict mode with info", + input: ` + eval_info instant at 3m min(baz) + {src="a"} 1 +`, + expected: ` + eval instant at 3m min(baz) + expect info + expect no_warn + {src="a"} 1 +`, + }, + { + name: "Strict mode with ordered", + input: ` + eval_ordered instant at 4m max(qux) + {src="a"} 1 +`, + expected: ` + eval instant at 4m max(qux) + expect ordered + expect no_warn + expect no_info + {src="a"} 1 +`, + }, + { + name: "Strict mode with multiple eval blocks", + input: ` + eval_fail instant at 1m sum(foo) + expected_fail_message something else went wrong + {src="a"} 1 + + eval_warn instant at 2m avg(bar) + {src="a"} 1 +`, + expected: ` + eval instant at 1m sum(foo) + expect fail msg:something else went wrong + expect no_warn + expect no_info + {src="a"} 1 + + eval instant at 2m avg(bar) + expect warn + expect no_info + {src="a"} 1 +`, + }, + { + name: "Strict mode with already migrated syntax (no changes)", + input: ` + eval instant at 1m sum(foo) + expect fail msg:something went wrong + expect no_warn + expect no_info + {src="a"} 1 +`, + expected: ` + eval instant at 1m sum(foo) + expect fail msg:something went wrong + expect no_warn + expect no_info + {src="a"} 1 +`, + }, + { + name: "Strict mode with only comments and whitespace", + input: ` + # This is a comment + +`, + expected: ` + # This is a comment + +`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assertMigration(t, "strict", tc.input, tc.expected) + }) + } +} + +func TestMigrateTestData_TolerantMode(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "Tolerant mode with fail", + input: ` + eval_fail instant at 1m sum(foo) + expected_fail_message something went wrong + {src="a"} 1 +`, + expected: ` + eval instant at 1m sum(foo) + expect fail msg:something went wrong + {src="a"} 1 +`, + }, + { + name: "Tolerant mode with warn", + input: ` + eval_warn instant at 2m avg(bar) + {src="a"} 1 +`, + expected: ` + eval instant at 2m avg(bar) + {src="a"} 1 +`, + }, + { + name: "Tolerant mode with info", + input: ` + eval_info instant at 3m min(baz) + {src="a"} 1 +`, + expected: ` + eval instant at 3m min(baz) + {src="a"} 1 +`, + }, + { + name: "Tolerant mode with ordered", + input: ` + eval_ordered instant at 4m max(qux) + {src="a"} 1 +`, + expected: ` + eval instant at 4m max(qux) + expect ordered + {src="a"} 1 +`, + }, + { + name: "Tolerant mode with multiple eval blocks", + input: ` + eval_fail instant at 1m sum(foo) + expected_fail_message something else went wrong + {src="a"} 1 + + eval_warn instant at 2m avg(bar) + {src="a"} 1 +`, + expected: ` + eval instant at 1m sum(foo) + expect fail msg:something else went wrong + {src="a"} 1 + + eval instant at 2m avg(bar) + {src="a"} 1 +`, + }, + { + name: "Tolerant mode with already migrated syntax (no changes)", + input: ` + eval instant at 1m sum(foo) + expect fail msg:something went wrong + {src="a"} 1 +`, + expected: ` + eval instant at 1m sum(foo) + expect fail msg:something went wrong + {src="a"} 1 +`, + }, + { + name: "Tolerant mode with only comments and whitespace", + input: ` + # This is a comment + +`, + expected: ` + # This is a comment + +`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assertMigration(t, "tolerant", tc.input, tc.expected) + }) + } +}