Merge remote-tracking branch 'origin/main' into krajo/st-in-xorchunk

.gitattributes

@@ -0,0 +1 @@
+web/api/v1/testdata/openapi_golden.yaml linguist-generated

.github/PULL_REQUEST_TEMPLATE.md

@@ -28,6 +28,7 @@ If no, just write "NONE" in the release-notes block below.
 Otherwise, please describe what should be mentioned in the CHANGELOG. Use the following prefixes:
 [FEATURE] [ENHANCEMENT] [PERF] [BUGFIX] [SECURITY] [CHANGE]
 Refer to the existing CHANGELOG for inspiration:  https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md
+A concrete example may look as follows (be sure to leave out the surrounding quotes): "[FEATURE] API: Add /api/v1/features for clients to understand which features are supported".
 If you need help formulating your entries, consult the reviewer(s).
 -->
 ```release-notes

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/setup_environment
         with:
           enable_npm: true
@@ -37,7 +37,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/setup_environment
       - run: go test --tags=dedupelabels ./...
       - run: go test --tags=slicelabels -race ./cmd/prometheus ./model/textparse ./prompb/...
@@ -81,7 +81,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/setup_environment
         with:
           enable_go: false
@@ -146,7 +146,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/build
         with:
           promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386"
@@ -173,7 +173,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/build
         with:
           parallelism: 12
@@ -212,7 +212,7 @@ jobs:
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/setup_environment
         with:
           enable_npm: true
@@ -270,7 +270,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/publish_main
         with:
           docker_hub_login: ${{ secrets.docker_hub_login }}
@@ -289,7 +289,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/publish_release
         with:
           docker_hub_login: ${{ secrets.docker_hub_login }}
@@ -306,7 +306,7 @@ jobs:
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - name: Install nodejs
         uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
         with:

.golangci.yml

@@ -124,6 +124,8 @@ linters:
         # Disable this check for now since it introduces too many changes in our existing codebase.
         # See https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/modernize#hdr-Analyzer_omitzero for more details.
         - omitzero
+        # Disable waitgroup check until we really move to Go 1.25.
+        - waitgroup
     perfsprint:
       # Optimizes even if it requires an int or uint type cast.
       int-conversion: true

.yamllint

@@ -2,6 +2,7 @@
 extends: default
 ignore: |
   **/node_modules
+  web/api/v1/testdata/openapi_*_golden.yaml
 
 rules:
   braces:

CODEOWNERS

@@ -2,25 +2,28 @@
 # Please keep this file in sync with the MAINTAINERS.md file!
 #
 
+# Prometheus team members are members of the "default maintainers" github team.
+# They are code owners by default for the whole repo.
+* @prometheus/default-maintainers
+
 # Subsystems.
-/Makefile @simonpasquier @SuperQ
-/cmd/promtool @dgl
-/documentation/prometheus-mixin @metalmatze
-/model/histogram @beorn7 @krajorama
-/web/ui @juliusv
-/web/ui/module @juliusv @nexucis
-/promql @roidelapluie
-/storage/remote @cstyan @bwplotka @tomwilkie @npazosmendez @alexgreenbank
-/storage/remote/otlptranslator @aknuds1 @jesusvazquez @ArthurSens
-/tsdb @jesusvazquez @codesome @bwplotka @krajorama
+/Makefile                       @prometheus/default-maintainers @simonpasquier @SuperQ
+/cmd/promtool                   @prometheus/default-maintainers @dgl
+/documentation/prometheus-mixin @prometheus/default-maintainers @metalmatze
+/model/histogram                @prometheus/default-maintainers @beorn7 @krajorama
+/web/ui                         @prometheus/default-maintainers @juliusv
+/web/ui/module                  @prometheus/default-maintainers @juliusv @nexucis
+/promql                         @prometheus/default-maintainers @roidelapluie
+/storage/remote                 @prometheus/default-maintainers @cstyan @bwplotka @tomwilkie @alexgreenbank
+/storage/remote/otlptranslator  @prometheus/default-maintainers @aknuds1 @jesusvazquez @ArthurSens
+/tsdb                           @prometheus/default-maintainers @jesusvazquez @codesome @bwplotka @krajorama
 
 # Service discovery.
-/discovery/kubernetes @brancz
-/discovery/stackit @jkroepke
+/discovery/kubernetes           @prometheus/default-maintainers @brancz
+/discovery/stackit              @prometheus/default-maintainers @jkroepke
+/discovery/aws/                 @prometheus/default-maintainers @matt-gp @sysadmind
 # Pending
-# https://github.com/prometheus/prometheus/pull/17105#issuecomment-3248209452
-# /discovery/aws/ @matt-gp @sysadmind
 # https://github.com/prometheus/prometheus/pull/15212#issuecomment-3575225179
-# /discovery/aliyun @KeyOfSpectator
+# /discovery/aliyun               @prometheus/default-maintainers @KeyOfSpectator
 # https://github.com/prometheus/prometheus/pull/14108#issuecomment-2639515421
-# /discovery/nomad @jaloren @jrasell
+# /discovery/nomad                @prometheus/default-maintainers @jaloren @jrasell

MAINTAINERS.md

@@ -18,7 +18,7 @@ Maintainers for specific parts of the codebase:
 * `model/histogram` and other code related to native histograms: Björn Rabenstein (<beorn@grafana.com> / @beorn7),
 George Krajcsovits (<gyorgy.krajcsovits@grafana.com> / @krajorama)
 * `storage`
-  * `remote`: Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (tom.wilkie@gmail.com / @tomwilkie), Nicolás Pazos (<npazosmendez@gmail.com> / @npazosmendez), Alex Greenbank (<alexgreenbank@yahoo.com> / @alexgreenbank)
+  * `remote`: Callum Styan (<callumstyan@gmail.com> / @cstyan), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Tom Wilkie (tom.wilkie@gmail.com / @tomwilkie), Alex Greenbank (<alexgreenbank@yahoo.com> / @alexgreenbank)
     * `otlptranslator`: Arthur Silva Sens (<arthursens2005@gmail.com> / @ArthurSens), Arve Knudsen (<arve.knudsen@gmail.com> / @aknuds1), Jesús Vázquez (<jesus.vazquez@grafana.com> / @jesusvazquez)
 * `tsdb`: Ganesh Vernekar (<ganesh@grafana.com> / @codesome), Bartłomiej Płotka (<bwplotka@gmail.com> / @bwplotka), Jesús Vázquez (<jesus.vazquez@grafana.com> / @jesusvazquez), George Krajcsovits (<gyorgy.krajcsovits@grafana.com> / @krajorama)
 * `web`

Makefile.common

@@ -292,6 +292,16 @@ $(PUBLISH_DOCKER_ARCHS): common-docker-publish-%:
 			echo "Pushing default variant ($$variant_name) for linux-$*"; \
 			docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)"; \
 		fi; \
+		if [ "$(DOCKER_IMAGE_TAG)" = "latest" ]; then \
+			if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \
+				echo "Pushing $$variant_name variant version tags for linux-$*"; \
+				docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \
+			fi; \
+			if [ "$$dockerfile" = "Dockerfile" ]; then \
+				echo "Pushing default variant version tag for linux-$*"; \
+				docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"; \
+			fi; \
+		fi; \
 	done
 
 DOCKER_MAJOR_VERSION_TAG = $(firstword $(subst ., ,$(shell cat VERSION)))
@@ -328,6 +338,18 @@ common-docker-manifest:
 			DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG)); \
 			DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)"; \
 		fi; \
+		if [ "$(DOCKER_IMAGE_TAG)" = "latest" ]; then \
+			if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \
+				echo "Creating manifest for $$variant_name variant version tag"; \
+				DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name); \
+				DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \
+			fi; \
+			if [ "$$dockerfile" = "Dockerfile" ]; then \
+				echo "Creating default variant version tag manifest"; \
+				DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):v$(DOCKER_MAJOR_VERSION_TAG)); \
+				DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)"; \
+			fi; \
+		fi; \
 	done
 
 .PHONY: promu

cmd/prometheus/main.go

@@ -265,13 +265,26 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error {
 			case "ooo-native-histograms":
 				logger.Warn("This option for --enable-feature is now permanently enabled and therefore a no-op.", "option", o)
 			case "created-timestamp-zero-ingestion":
+				// NOTE(bwplotka): Once AppendableV1 is removed, there will be only the TSDB and agent flags.
 				c.scrape.EnableStartTimestampZeroIngestion = true
 				c.web.STZeroIngestionEnabled = true
+				c.tsdb.EnableSTAsZeroSample = true
 				c.agent.EnableSTAsZeroSample = true
+
 				// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
+				// This is to widen the ST support surface.
 				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
 				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
-				logger.Info("Experimental created timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
+				logger.Info("Experimental start timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
+			case "st-storage":
+				// TODO(bwplotka): Implement ST Storage as per PROM-60 and document this hidden feature flag.
+				c.tsdb.EnableSTStorage = true
+				c.agent.EnableSTStorage = true
+
+				// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers. This is to widen the ST support surface.
+				config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
+				logger.Info("Experimental start timestamp storage enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
 			case "delayed-compaction":
 				c.tsdb.EnableDelayedCompaction = true
 				logger.Info("Experimental delayed compaction is enabled.")
@@ -692,6 +705,7 @@ func main() {
 	}
 	if cfgFile.StorageConfig.TSDBConfig != nil {
 		cfg.tsdb.OutOfOrderTimeWindow = cfgFile.StorageConfig.TSDBConfig.OutOfOrderTimeWindow
+		cfg.tsdb.StaleSeriesCompactionThreshold = cfgFile.StorageConfig.TSDBConfig.StaleSeriesCompactionThreshold
 		if cfgFile.StorageConfig.TSDBConfig.Retention != nil {
 			if cfgFile.StorageConfig.TSDBConfig.Retention.Time > 0 {
 				cfg.tsdb.RetentionDuration = cfgFile.StorageConfig.TSDBConfig.Retention.Time
@@ -871,16 +885,29 @@ func main() {
 		os.Exit(1)
 	}
 
-	scrapeManager, err := scrape.NewManager(
-		&cfg.scrape,
-		logger.With("component", "scrape manager"),
-		logging.NewJSONFileLogger,
-		fanoutStorage,
-		prometheus.DefaultRegisterer,
-	)
-	if err != nil {
-		logger.Error("failed to create a scrape manager", "err", err)
-		os.Exit(1)
+	var scrapeManager *scrape.Manager
+	{
+		// TODO(bwplotka): Switch to AppendableV2 by default.
+		// See: https://github.com/prometheus/prometheus/issues/17632
+		var (
+			scrapeAppendable   storage.Appendable = fanoutStorage
+			scrapeAppendableV2 storage.AppendableV2
+		)
+		if cfg.tsdb.EnableSTStorage {
+			scrapeAppendable = nil
+			scrapeAppendableV2 = fanoutStorage
+		}
+		scrapeManager, err = scrape.NewManager(
+			&cfg.scrape,
+			logger.With("component", "scrape manager"),
+			logging.NewJSONFileLogger,
+			scrapeAppendable, scrapeAppendableV2,
+			prometheus.DefaultRegisterer,
+		)
+		if err != nil {
+			logger.Error("failed to create a scrape manager", "err", err)
+			os.Exit(1)
+		}
 	}
 
 	var (
@@ -1367,6 +1394,8 @@ func main() {
 					"WALSegmentSize", cfg.tsdb.WALSegmentSize,
 					"WALCompressionType", cfg.tsdb.WALCompressionType,
 					"BlockReloadInterval", cfg.tsdb.BlockReloadInterval,
+					"EnableSTAsZeroSample", cfg.tsdb.EnableSTAsZeroSample,
+					"EnableSTStorage", cfg.tsdb.EnableSTStorage,
 				)
 
 				startTimeMargin := int64(2 * time.Duration(cfg.tsdb.MinBlockDuration).Seconds() * 1000)
@@ -1424,6 +1453,7 @@ func main() {
 					"MaxWALTime", cfg.agent.MaxWALTime,
 					"OutOfOrderTimeWindow", cfg.agent.OutOfOrderTimeWindow,
 					"EnableSTAsZeroSample", cfg.agent.EnableSTAsZeroSample,
+					"EnableSTStorage", cfg.tsdb.EnableSTStorage,
 				)
 
 				localStorage.Set(db, 0)
@@ -1943,6 +1973,9 @@ type tsdbOptions struct {
 	UseUncachedIO                  bool
 	BlockCompactionExcludeFunc     tsdb.BlockExcludeFilterFunc
 	BlockReloadInterval            model.Duration
+	EnableSTAsZeroSample           bool
+	EnableSTStorage                bool
+	StaleSeriesCompactionThreshold float64
 }
 
 func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
@@ -1969,6 +2002,9 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
 		BlockCompactionExcludeFunc:     opts.BlockCompactionExcludeFunc,
 		BlockReloadInterval:            time.Duration(opts.BlockReloadInterval),
 		FeatureRegistry:                features.DefaultRegistry,
+		EnableSTAsZeroSample:           opts.EnableSTAsZeroSample,
+		EnableSTStorage:                opts.EnableSTStorage,
+		StaleSeriesCompactionThreshold: opts.StaleSeriesCompactionThreshold,
 	}
 }
 
@@ -1983,6 +2019,7 @@ type agentOptions struct {
 	NoLockfile             bool
 	OutOfOrderTimeWindow   int64 // TODO(bwplotka): Unused option, fix it or remove.
 	EnableSTAsZeroSample   bool
+	EnableSTStorage        bool
 }
 
 func (opts agentOptions) ToAgentOptions(outOfOrderTimeWindow int64) agent.Options {
@@ -1999,6 +2036,7 @@ func (opts agentOptions) ToAgentOptions(outOfOrderTimeWindow int64) agent.Option
 		NoLockfile:           opts.NoLockfile,
 		OutOfOrderTimeWindow: outOfOrderTimeWindow,
 		EnableSTAsZeroSample: opts.EnableSTAsZeroSample,
+		EnableSTStorage:      opts.EnableSTStorage,
 	}
 }
 

cmd/prometheus/query_log_test.go

@@ -334,7 +334,8 @@ func (p *queryLogTest) run(t *testing.T) {
 
 	p.query(t)
 
-	ql := readQueryLog(t, queryLogFile.Name())
+	// Wait for query log entry to be written (avoid race with file I/O).
+	ql := waitForQueryLog(t, queryLogFile.Name(), 1)
 	qc := len(ql)
 	if p.exactQueryCount() {
 		require.Equal(t, 1, qc)
@@ -361,7 +362,8 @@ func (p *queryLogTest) run(t *testing.T) {
 	p.query(t)
 	qc++
 
-	ql = readQueryLog(t, queryLogFile.Name())
+	// Wait for query log entry to be written (avoid race with file I/O).
+	ql = waitForQueryLog(t, queryLogFile.Name(), qc)
 	if p.exactQueryCount() {
 		require.Len(t, ql, qc)
 	} else {
@@ -392,7 +394,8 @@ func (p *queryLogTest) run(t *testing.T) {
 
 	qc++
 
-	ql = readQueryLog(t, newFile.Name())
+	// Wait for query log entry to be written (avoid race with file I/O).
+	ql = waitForQueryLog(t, newFile.Name(), qc)
 	if p.exactQueryCount() {
 		require.Len(t, ql, qc)
 	} else {
@@ -404,7 +407,8 @@ func (p *queryLogTest) run(t *testing.T) {
 
 	p.query(t)
 
-	ql = readQueryLog(t, queryLogFile.Name())
+	// Wait for query log entry to be written (avoid race with file I/O).
+	ql = waitForQueryLog(t, queryLogFile.Name(), 1)
 	qc = len(ql)
 	if p.exactQueryCount() {
 		require.Equal(t, 1, qc)
@@ -446,6 +450,18 @@ func readQueryLog(t *testing.T, path string) []queryLogLine {
 	return ql
 }
 
+// waitForQueryLog waits for the query log to contain at least minEntries entries,
+// polling at regular intervals until the timeout is reached.
+func waitForQueryLog(t *testing.T, path string, minEntries int) []queryLogLine {
+	t.Helper()
+	var ql []queryLogLine
+	require.Eventually(t, func() bool {
+		ql = readQueryLog(t, path)
+		return len(ql) >= minEntries
+	}, 5*time.Second, 100*time.Millisecond, "timed out waiting for query log to have at least %d entries, got %d", minEntries, len(ql))
+	return ql
+}
+
 func TestQueryLog(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode.")

cmd/prometheus/testdata/features.json

@@ -4,6 +4,8 @@
     "exclude_alerts": true,
     "label_values_match": true,
     "lifecycle": false,
+    "openapi_3.1": true,
+    "openapi_3.2": true,
     "otlp_write_receiver": false,
     "query_stats": true,
     "query_warnings": true,

cmd/promtool/main_test.go

@@ -734,7 +734,6 @@ func TestTSDBDumpCommand(t *testing.T) {
 	load 1m
 		metric{foo="bar"} 1 2 3
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	for _, c := range []struct {
 		name           string

cmd/promtool/tsdb_test.go

@@ -97,7 +97,6 @@ func TestTSDBDump(t *testing.T) {
 			heavy_metric{foo="bar"} 5 4 3 2 1
 			heavy_metric{foo="foo"} 5 4 3 2 1
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	tests := []struct {
 		name           string
@@ -196,7 +195,6 @@ func TestTSDBDumpOpenMetrics(t *testing.T) {
 			my_counter{foo="bar", baz="abc"} 1 2 3 4 5
 			my_gauge{bar="foo", abc="baz"} 9 8 0 4 7
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	tests := []struct {
 		name           string

config/config.go

@@ -1107,6 +1107,10 @@ type TSDBConfig struct {
 	// This should not be used directly and must be converted into OutOfOrderTimeWindow.
 	OutOfOrderTimeWindowFlag model.Duration `yaml:"out_of_order_time_window,omitempty"`
 
+	// StaleSeriesCompactionThreshold is a number between 0.0-1.0 indicating the % of stale series in
+	// the in-memory Head block. If the % of stale series crosses this threshold, stale series compaction is run immediately.
+	StaleSeriesCompactionThreshold float64 `yaml:"stale_series_compaction_threshold,omitempty"`
+
 	Retention *TSDBRetentionConfig `yaml:"retention,omitempty"`
 }
 

config/config_test.go

@@ -1733,8 +1733,9 @@ var expectedConf = &Config{
 	},
 	StorageConfig: StorageConfig{
 		TSDBConfig: &TSDBConfig{
-			OutOfOrderTimeWindow:     30 * time.Minute.Milliseconds(),
-			OutOfOrderTimeWindowFlag: model.Duration(30 * time.Minute),
+			OutOfOrderTimeWindow:           30 * time.Minute.Milliseconds(),
+			OutOfOrderTimeWindowFlag:       model.Duration(30 * time.Minute),
+			StaleSeriesCompactionThreshold: 0.5,
 			Retention: &TSDBRetentionConfig{
 				Time: model.Duration(24 * time.Hour),
 				Size: 1 * units.GiB,

config/testdata/conf.good.yml

@@ -453,6 +453,7 @@ alerting:
 storage:
   tsdb:
     out_of_order_time_window: 30m
+    stale_series_compaction_threshold: 0.5
     retention:
       time: 1d
       size: 1GB

discovery/aws/aws.go

@@ -101,7 +101,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	switch c.Role {
 	case RoleEC2:
 		if c.EC2SDConfig == nil {
-			c.EC2SDConfig = &DefaultEC2SDConfig
+			ec2Config := DefaultEC2SDConfig
+			c.EC2SDConfig = &ec2Config
 		}
 		c.EC2SDConfig.HTTPClientConfig = c.HTTPClientConfig
 		if c.Region != "" {
@@ -133,7 +134,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error {
 		}
 	case RoleECS:
 		if c.ECSSDConfig == nil {
-			c.ECSSDConfig = &DefaultECSSDConfig
+			ecsConfig := DefaultECSSDConfig
+			c.ECSSDConfig = &ecsConfig
 		}
 		c.ECSSDConfig.HTTPClientConfig = c.HTTPClientConfig
 		if c.Region != "" {
@@ -165,7 +167,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error {
 		}
 	case RoleLightsail:
 		if c.LightsailSDConfig == nil {
-			c.LightsailSDConfig = &DefaultLightsailSDConfig
+			lightsailConfig := DefaultLightsailSDConfig
+			c.LightsailSDConfig = &lightsailConfig
 		}
 		c.LightsailSDConfig.HTTPClientConfig = c.HTTPClientConfig
 		if c.Region != "" {

discovery/aws/aws_test.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v3"
+	"go.yaml.in/yaml/v3"
 )
 
 func TestRoleUnmarshalYAML(t *testing.T) {
@@ -177,3 +177,109 @@ port: 9300`,
 		})
 	}
 }
+
+// TestMultipleSDConfigsDoNotShareState verifies that multiple AWS SD configs
+// don't share the same underlying configuration object. This was a bug where
+// all configs pointed to the same global default, causing port and other
+// settings from one job to overwrite settings in another job.
+func TestMultipleSDConfigsDoNotShareState(t *testing.T) {
+	tests := []struct {
+		name         string
+		yaml         string
+		validateFunc func(t *testing.T, cfg1, cfg2 *SDConfig)
+	}{
+		{
+			name: "EC2MultipleJobsDifferentPorts",
+			yaml: `
+- role: ec2
+  region: us-west-2
+  port: 9100
+  filters:
+    - name: tag:Name
+      values: [host-1]
+- role: ec2
+  region: us-west-2
+  port: 9101
+  filters:
+    - name: tag:Name
+      values: [host-2]`,
+			validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) {
+				require.Equal(t, RoleEC2, cfg1.Role)
+				require.Equal(t, RoleEC2, cfg2.Role)
+				require.NotNil(t, cfg1.EC2SDConfig)
+				require.NotNil(t, cfg2.EC2SDConfig)
+
+				// Verify ports are different and not shared
+				require.Equal(t, 9100, cfg1.EC2SDConfig.Port)
+				require.Equal(t, 9101, cfg2.EC2SDConfig.Port)
+
+				// Verify filters are different and not shared
+				require.Len(t, cfg1.EC2SDConfig.Filters, 1)
+				require.Len(t, cfg2.EC2SDConfig.Filters, 1)
+				require.Equal(t, []string{"host-1"}, cfg1.EC2SDConfig.Filters[0].Values)
+				require.Equal(t, []string{"host-2"}, cfg2.EC2SDConfig.Filters[0].Values)
+
+				// Most importantly: verify they're not the same pointer
+				require.NotSame(t, cfg1.EC2SDConfig, cfg2.EC2SDConfig,
+					"EC2SDConfig objects should not share the same memory address")
+			},
+		},
+		{
+			name: "ECSMultipleJobsDifferentPorts",
+			yaml: `
+- role: ecs
+  region: us-east-1
+  port: 8080
+  clusters: [cluster-a]
+- role: ecs
+  region: us-east-1
+  port: 8081
+  clusters: [cluster-b]`,
+			validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) {
+				require.Equal(t, RoleECS, cfg1.Role)
+				require.Equal(t, RoleECS, cfg2.Role)
+				require.NotNil(t, cfg1.ECSSDConfig)
+				require.NotNil(t, cfg2.ECSSDConfig)
+
+				require.Equal(t, 8080, cfg1.ECSSDConfig.Port)
+				require.Equal(t, 8081, cfg2.ECSSDConfig.Port)
+				require.Equal(t, []string{"cluster-a"}, cfg1.ECSSDConfig.Clusters)
+				require.Equal(t, []string{"cluster-b"}, cfg2.ECSSDConfig.Clusters)
+
+				require.NotSame(t, cfg1.ECSSDConfig, cfg2.ECSSDConfig,
+					"ECSSDConfig objects should not share the same memory address")
+			},
+		},
+		{
+			name: "LightsailMultipleJobsDifferentPorts",
+			yaml: `
+- role: lightsail
+  region: eu-west-1
+  port: 7070
+- role: lightsail
+  region: eu-west-1
+  port: 7071`,
+			validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) {
+				require.Equal(t, RoleLightsail, cfg1.Role)
+				require.Equal(t, RoleLightsail, cfg2.Role)
+				require.NotNil(t, cfg1.LightsailSDConfig)
+				require.NotNil(t, cfg2.LightsailSDConfig)
+
+				require.Equal(t, 7070, cfg1.LightsailSDConfig.Port)
+				require.Equal(t, 7071, cfg2.LightsailSDConfig.Port)
+
+				require.NotSame(t, cfg1.LightsailSDConfig, cfg2.LightsailSDConfig,
+					"LightsailSDConfig objects should not share the same memory address")
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var configs []SDConfig
+			require.NoError(t, yaml.Unmarshal([]byte(tt.yaml), &configs))
+			require.Len(t, configs, 2)
+			tt.validateFunc(t, &configs[0], &configs[1])
+		})
+	}
+}

docs/configuration/configuration.md

@@ -3496,6 +3496,19 @@ with this feature.
 # to the timestamp of the last appended sample for the same series.
 [ out_of_order_time_window: <duration> | default = 0s ]
 
+# Configures the trigger point for compacting the stale series from the memory into persistent blocks
+# and remove those stale series from the memory.
+#
+# The threshold is a number between 0.0 and 1.0. It represents the ratio of stale series in the memory
+# to the total series in the memory. The stale series compaction is triggered when this ratio crosses
+# the configured threshold. It may not trigger the stale series compaction if the usual head compaction
+# is about to happen soon.
+#
+# If set to 0, stale series compaction is disabled.
+#
+# This is an experimental feature, this behaviour could change or be removed in the future.
+[ stale_series_compaction_threshold: <float> | default = 0 ]
+
 
 # Configures data retention settings for TSDB.
 #

docs/querying/api.md

@@ -6,6 +6,22 @@ sort_rank: 7
 The current stable HTTP API is reachable under `/api/v1` on a Prometheus
 server. Any non-breaking additions will be added under that endpoint.
 
+## OpenAPI Specification
+
+An OpenAPI specification for the HTTP API is available at `/api/v1/openapi.yaml`.
+By default, it returns OpenAPI 3.1 for broader compatibility. Use `?openapi_version=3.2`
+for OpenAPI 3.2, which includes advanced features and endpoints like `/api/v1/notifications/live`.
+
+This machine-readable specification describes all available endpoints, request parameters,
+response formats, and schemas.
+
+The OpenAPI specification can be used to:
+
+- Generate client libraries in various programming languages.
+- Validate API requests and responses.
+- Generate interactive API documentation.
+- Test API endpoints.
+
 ## Format overview
 
 The API response format is JSON. Every successful API request returns a `2xx`

docs/querying/functions.md

@@ -568,6 +568,8 @@ While `info` normally automatically finds all matching info series, it's possibl
 restrict them by providing a `__name__` label matcher, e.g.
 `{__name__="target_info"}`.
 
+Note that if there are any time series in `v` that match the `data-label-selector` (or the default `target_info` if that argument is not specified), they will be treated as info series and will be returned unchanged.
+
 ### Limitations
 
 In its current iteration, `info` defaults to considering only info series with

documentation/examples/remote_storage/go.mod

@@ -1,6 +1,6 @@
 module github.com/prometheus/prometheus/documentation/examples/remote_storage
 
-go 1.24.0
+go 1.25.0
 
 require (
 	github.com/alecthomas/kingpin/v2 v2.4.0

go.mod

@@ -1,6 +1,6 @@
 module github.com/prometheus/prometheus
 
-go 1.24.0
+go 1.25.0
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0
@@ -54,6 +54,8 @@ require (
 	github.com/oklog/ulid/v2 v2.1.1
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.142.0
 	github.com/ovh/go-ovh v1.9.0
+	github.com/pb33f/libopenapi v0.31.1
+	github.com/pb33f/libopenapi-validator v0.10.0
 	github.com/prometheus/alertmanager v0.30.0
 	github.com/prometheus/client_golang v1.23.2
 	github.com/prometheus/client_golang/exp v0.0.0-20260101091701-2cd067eb23c9
@@ -84,6 +86,8 @@ require (
 	go.uber.org/automaxprocs v1.6.0
 	go.uber.org/goleak v1.3.0
 	go.yaml.in/yaml/v2 v2.4.3
+	go.yaml.in/yaml/v3 v3.0.4
+	go.yaml.in/yaml/v4 v4.0.0-rc.3
 	golang.org/x/oauth2 v0.34.0
 	golang.org/x/sync v0.19.0
 	golang.org/x/sys v0.39.0
@@ -102,6 +106,9 @@ require (
 
 require (
 	github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/basgys/goxml2json v1.1.1-0.20231018121955-e66ee54ceaad // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/go-openapi/swag/cmdutils v0.25.4 // indirect
 	github.com/go-openapi/swag/conv v0.25.4 // indirect
 	github.com/go-openapi/swag/fileutils v0.25.4 // indirect
@@ -113,8 +120,10 @@ require (
 	github.com/go-openapi/swag/stringutils v0.25.4 // indirect
 	github.com/go-openapi/swag/typeutils v0.25.4 // indirect
 	github.com/go-openapi/swag/yamlutils v0.25.4 // indirect
+	github.com/pb33f/jsonpath v0.7.0 // indirect
+	github.com/pb33f/ordered-map/v2 v2.3.0 // indirect
+	github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
 )
 
@@ -237,7 +246,7 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.0.3 // indirect
 	k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
 	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect

go.sum

@@ -81,6 +81,10 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
 github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
 github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
+github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
+github.com/basgys/goxml2json v1.1.1-0.20231018121955-e66ee54ceaad h1:3swAvbzgfaI6nKuDDU7BiKfZRdF+h2ZwKgMHd8Ha4t8=
+github.com/basgys/goxml2json v1.1.1-0.20231018121955-e66ee54ceaad/go.mod h1:9+nBLYNWkvPcq9ep0owWUsPTLgL9ZXTsZWcCSVGGLJ0=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -88,6 +92,10 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow=
+github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -116,6 +124,8 @@ github.com/digitalocean/godo v1.171.0 h1:QwpkwWKr3v7yxc8D4NQG973NoR9APCEWjYnLOQe
 github.com/digitalocean/godo v1.171.0/go.mod h1:xQsWpVCCbkDrWisHA72hPzPlnC+4W5w/McZY5ij9uvU=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
+github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
 github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
@@ -437,6 +447,14 @@ github.com/ovh/go-ovh v1.9.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pb33f/jsonpath v0.7.0 h1:3oG6yu1RqNoMZpqnRjBMqi8fSIXWoDAKDrsB0QGTcoU=
+github.com/pb33f/jsonpath v0.7.0/go.mod h1:/+JlSIjWA2ijMVYGJ3IQPF4Q1nLMYbUTYNdk0exCDPQ=
+github.com/pb33f/libopenapi v0.31.1 h1:smGr45U2Y+hHWYKiEV13oS2tP9IUnscqNb5qsvT9+YI=
+github.com/pb33f/libopenapi v0.31.1/go.mod h1:oaebeA5l58AFbZ7qRKTtMnu15JEiPlaBas1vLDcw9vs=
+github.com/pb33f/libopenapi-validator v0.10.0 h1:9XhgxW2jTDd+1aDMuIjGUsWaeUaPi5ql2z1Y+WBltiE=
+github.com/pb33f/libopenapi-validator v0.10.0/go.mod h1:hW3wIpg4YCxLrJxyTrfrzP9Mtt9FvbD/nm0yemUcjSs=
+github.com/pb33f/ordered-map/v2 v2.3.0 h1:k2OhVEQkhTCQMhAicQ3Z6iInzoZNQ7L9MVomwKBZ5WQ=
+github.com/pb33f/ordered-map/v2 v2.3.0/go.mod h1:oe5ue+6ZNhy7QN9cPZvPA23Hx0vMHnNVeMg4fGdCANw=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
@@ -491,6 +509,8 @@ github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPK
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
+github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.36 h1:ObX9hZmK+VmijreZO/8x9pQ8/P/ToHD/bdSb4Eg4tUo=
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.36/go.mod h1:LEsDu4BubxK7/cWhtlQWfuxwL4rf/2UEpxXz1o1EMtM=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
@@ -517,6 +537,7 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@@ -533,6 +554,7 @@ github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
 go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
@@ -620,12 +642,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
 golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/exp v0.0.0-20250808145144-a408d31f581a h1:Y+7uR/b1Mw2iSXZ3G//1haIiSElDQZ8KWh0h+sZPG90=
 golang.org/x/exp v0.0.0-20250808145144-a408d31f581a/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
 golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -638,6 +664,10 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
 golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
 golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
@@ -648,6 +678,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
 golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -667,23 +699,37 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
 golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
 golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
 golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
@@ -694,6 +740,8 @@ golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
 golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=

go.work

@@ -1,4 +1,4 @@
-go 1.24.0
+go 1.25.0
 
 use (
 	.

internal/tools/go.mod

@@ -1,6 +1,6 @@
 module github.com/prometheus/prometheus/internal/tools
 
-go 1.24.0
+go 1.25.0
 
 require (
 	github.com/bufbuild/buf v1.62.1

model/rulefmt/rulefmt.go

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	"github.com/prometheus/common/model"
-	"gopkg.in/yaml.v3"
+	"go.yaml.in/yaml/v3"
 
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/promql"

model/rulefmt/rulefmt_test.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v3"
+	"go.yaml.in/yaml/v3"
 )
 
 func TestParseFileSuccess(t *testing.T) {

notifier/manager.go

@@ -163,19 +163,33 @@ func (n *Manager) ApplyConfig(conf *config.Config) error {
 		if oldAmSet, ok := configToAlertmanagers[hash]; ok {
 			ams.ams = oldAmSet.ams
 			ams.droppedAms = oldAmSet.droppedAms
-			ams.sendLoops = oldAmSet.sendLoops
+			// Only transfer sendLoops to the first new config with this hash.
+			// Subsequent configs with the same hash should not share the sendLoops
+			// map reference, as that would cause shared mutable state between
+			// alertmanagerSets (cleanup in one would affect the other).
+			oldAmSet.mtx.Lock()
+			if oldAmSet.sendLoops != nil {
+				ams.mtx.Lock()
+				ams.sendLoops = oldAmSet.sendLoops
+				oldAmSet.sendLoops = nil
+				ams.mtx.Unlock()
+			}
+			oldAmSet.mtx.Unlock()
 		}
 
 		amSets[k] = ams
 	}
 
-	// Clean up the send loops of sets that don't exist in the new config.
-	for k, oldAmSet := range n.alertmanagers {
-		if _, exists := amSets[k]; !exists {
-			oldAmSet.mtx.Lock()
+	// Clean up sendLoops that weren't transferred to new config.
+	// This happens when: (1) key was removed, or (2) key exists but hash changed.
+	// After the transfer loop above, any oldAmSet with non-nil sendLoops
+	// had its sendLoops NOT transferred (since we set it to nil on transfer).
+	for _, oldAmSet := range n.alertmanagers {
+		oldAmSet.mtx.Lock()
+		if oldAmSet.sendLoops != nil {
 			oldAmSet.cleanSendLoops(oldAmSet.ams...)
-			oldAmSet.mtx.Unlock()
 		}
+		oldAmSet.mtx.Unlock()
 	}
 
 	n.alertmanagers = amSets

promql/bench_test.go

@@ -338,7 +338,7 @@ func BenchmarkRangeQuery(b *testing.B) {
 	})
 	stor := teststorage.New(b)
 	stor.DisableCompactions() // Don't want auto-compaction disrupting timings.
-	defer stor.Close()
+
 	opts := promql.EngineOpts{
 		Logger:     nil,
 		Reg:        nil,
@@ -383,7 +383,6 @@ func BenchmarkRangeQuery(b *testing.B) {
 func BenchmarkJoinQuery(b *testing.B) {
 	stor := teststorage.New(b)
 	stor.DisableCompactions() // Don't want auto-compaction disrupting timings.
-	defer stor.Close()
 
 	opts := promql.EngineOpts{
 		Logger:     nil,
@@ -445,7 +444,6 @@ func BenchmarkJoinQuery(b *testing.B) {
 
 func BenchmarkNativeHistograms(b *testing.B) {
 	testStorage := teststorage.New(b)
-	defer testStorage.Close()
 
 	app := testStorage.Appender(context.TODO())
 	if err := generateNativeHistogramSeries(app, 3000); err != nil {
@@ -523,7 +521,6 @@ func BenchmarkNativeHistograms(b *testing.B) {
 
 func BenchmarkNativeHistogramsCustomBuckets(b *testing.B) {
 	testStorage := teststorage.New(b)
-	defer testStorage.Close()
 
 	app := testStorage.Appender(context.TODO())
 	if err := generateNativeHistogramCustomBucketsSeries(app, 3000); err != nil {
@@ -594,7 +591,6 @@ func BenchmarkNativeHistogramsCustomBuckets(b *testing.B) {
 func BenchmarkInfoFunction(b *testing.B) {
 	// Initialize test storage and generate test series data.
 	testStorage := teststorage.New(b)
-	defer testStorage.Close()
 
 	start := time.Unix(0, 0)
 	end := start.Add(2 * time.Hour)

promql/engine_test.go

@@ -676,7 +676,6 @@ func TestEngineEvalStmtTimestamps(t *testing.T) {
 load 10s
   metric 1 2
 `)
-	t.Cleanup(func() { storage.Close() })
 
 	cases := []struct {
 		Query       string
@@ -789,7 +788,6 @@ load 10s
   metricWith3SampleEvery10Seconds{a="3",b="2"} 1+1x100
   metricWith1HistogramEvery10Seconds {{schema:1 count:5 sum:20 buckets:[1 2 1 1]}}+{{schema:1 count:10 sum:5 buckets:[1 2 3 4]}}x100
 `)
-	t.Cleanup(func() { storage.Close() })
 
 	cases := []struct {
 		Query               string
@@ -1339,7 +1337,6 @@ load 10s
   bigmetric{a="1"} 1+1x100
   bigmetric{a="2"} 1+1x100
 `)
-	t.Cleanup(func() { storage.Close() })
 
 	// These test cases should be touching the limit exactly (hence no exceeding).
 	// Exceeding the limit will be tested by doing -1 to the MaxSamples.
@@ -1523,7 +1520,6 @@ func TestExtendedRangeSelectors(t *testing.T) {
 		withreset 1+1x4 1+1x5
 		notregular 0 5 100 2 8
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	tc := []struct {
 		query    string
@@ -1677,7 +1673,6 @@ load 10s
 load 1ms
   metric_ms 0+1x10000
 `)
-	t.Cleanup(func() { storage.Close() })
 
 	lbls1 := labels.FromStrings("__name__", "metric", "job", "1")
 	lbls2 := labels.FromStrings("__name__", "metric", "job", "2")
@@ -2283,7 +2278,6 @@ func TestSubquerySelector(t *testing.T) {
 		t.Run("", func(t *testing.T) {
 			engine := newTestEngine(t)
 			storage := promqltest.LoadedStorage(t, tst.loadString)
-			t.Cleanup(func() { storage.Close() })
 
 			for _, c := range tst.cases {
 				t.Run(c.Query, func(t *testing.T) {
@@ -3410,7 +3404,6 @@ metric 0 1 2
 		t.Run(c.name, func(t *testing.T) {
 			engine := promqltest.NewTestEngine(t, false, c.engineLookback, promqltest.DefaultMaxSamplesPerQuery)
 			storage := promqltest.LoadedStorage(t, load)
-			t.Cleanup(func() { storage.Close() })
 
 			opts := promql.NewPrometheusQueryOpts(false, c.queryLookback)
 			qry, err := engine.NewInstantQuery(context.Background(), storage, opts, query, c.ts)
@@ -3444,7 +3437,7 @@ func TestHistogramCopyFromIteratorRegression(t *testing.T) {
 histogram {{sum:4 count:4 buckets:[2 2]}} {{sum:6 count:6 buckets:[3 3]}} {{sum:1 count:1 buckets:[1]}}
 `
 	storage := promqltest.LoadedStorage(t, load)
-	t.Cleanup(func() { storage.Close() })
+
 	engine := promqltest.NewTestEngine(t, false, 0, promqltest.DefaultMaxSamplesPerQuery)
 
 	verify := func(t *testing.T, qry promql.Query, expected []histogram.FloatHistogram) {

promql/functions_test.go

@@ -33,7 +33,7 @@ func TestDeriv(t *testing.T) {
 	// This requires more precision than the usual test system offers,
 	// so we test it by hand.
 	storage := teststorage.New(t)
-	defer storage.Close()
+
 	opts := promql.EngineOpts{
 		Logger:     nil,
 		Reg:        nil,

promql/parser/generated_parser.y

@@ -850,14 +850,15 @@ series_item     : BLANK
                 // Histogram descriptions (part of unit testing).
                 | histogram_series_value
                         {
-                        $$ = []SequenceValue{{Histogram:$1}}
+                        $$ = []SequenceValue{yylex.(*parser).newHistogramSequenceValue($1)}
                         }
                 | histogram_series_value TIMES uint
                         {
                         $$ = []SequenceValue{}
                         // Add an additional value for time 0, which we ignore in tests.
+                        sv := yylex.(*parser).newHistogramSequenceValue($1)
                         for i:=uint64(0); i <= $3; i++{
-                                $$ = append($$, SequenceValue{Histogram:$1})
+                                $$ = append($$, sv)
                                 //$1 += $2
                         }
                         }

promql/parser/generated_parser.y.go

@@ -1931,15 +1931,16 @@ yydefault:
 	case 170:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		{
-			yyVAL.series = []SequenceValue{{Histogram: yyDollar[1].histogram}}
+			yyVAL.series = []SequenceValue{yylex.(*parser).newHistogramSequenceValue(yyDollar[1].histogram)}
 		}
 	case 171:
 		yyDollar = yyS[yypt-3 : yypt+1]
 		{
 			yyVAL.series = []SequenceValue{}
 			// Add an additional value for time 0, which we ignore in tests.
+			sv := yylex.(*parser).newHistogramSequenceValue(yyDollar[1].histogram)
 			for i := uint64(0); i <= yyDollar[3].uint; i++ {
-				yyVAL.series = append(yyVAL.series, SequenceValue{Histogram: yyDollar[1].histogram})
+				yyVAL.series = append(yyVAL.series, sv)
 				//$1 += $2
 			}
 		}

promql/parser/parse.go

@@ -70,6 +70,11 @@ type parser struct {
 
 	generatedParserResult any
 	parseErrors           ParseErrors
+
+	// lastHistogramCounterResetHintSet is set to true when the most recently
+	// built histogram had a counter_reset_hint explicitly specified.
+	// This is used to populate CounterResetHintSet in SequenceValue.
+	lastHistogramCounterResetHintSet bool
 }
 
 type Opt func(p *parser)
@@ -237,6 +242,11 @@ type SequenceValue struct {
 	Value     float64
 	Omitted   bool
 	Histogram *histogram.FloatHistogram
+	// CounterResetHintSet is true if the counter reset hint was explicitly
+	// specified in the test file using counter_reset_hint:... syntax.
+	// This allows distinguishing between "no hint specified" (don't care)
+	// vs "counter_reset_hint:unknown" (verify it's unknown).
+	CounterResetHintSet bool
 }
 
 func (v SequenceValue) String() string {
@@ -504,25 +514,30 @@ func (p *parser) mergeMaps(left, right *map[string]any) (ret *map[string]any) {
 }
 
 func (p *parser) histogramsIncreaseSeries(base, inc *histogram.FloatHistogram, times uint64) ([]SequenceValue, error) {
-	return p.histogramsSeries(base, inc, times, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
+	// Capture the hint set flag immediately after inc histogram is built.
+	// The base histogram's hint set flag was already captured.
+	hintSet := p.lastHistogramCounterResetHintSet
+	return p.histogramsSeries(base, inc, times, hintSet, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
 		res, _, _, err := a.Add(b)
 		return res, err
 	})
 }
 
 func (p *parser) histogramsDecreaseSeries(base, inc *histogram.FloatHistogram, times uint64) ([]SequenceValue, error) {
-	return p.histogramsSeries(base, inc, times, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
+	// Capture the hint set flag immediately after inc histogram is built.
+	hintSet := p.lastHistogramCounterResetHintSet
+	return p.histogramsSeries(base, inc, times, hintSet, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
 		res, _, _, err := a.Sub(b)
 		return res, err
 	})
 }
 
-func (*parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uint64,
+func (*parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uint64, counterResetHintSet bool,
 	combine func(*histogram.FloatHistogram, *histogram.FloatHistogram) (*histogram.FloatHistogram, error),
 ) ([]SequenceValue, error) {
 	ret := make([]SequenceValue, times+1)
 	// Add an additional value (the base) for time 0, which we ignore in tests.
-	ret[0] = SequenceValue{Histogram: base}
+	ret[0] = SequenceValue{Histogram: base, CounterResetHintSet: counterResetHintSet}
 	cur := base
 	for i := uint64(1); i <= times; i++ {
 		if cur.Schema > inc.Schema {
@@ -534,7 +549,7 @@ func (*parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uint6
 		if err != nil {
 			return ret, err
 		}
-		ret[i] = SequenceValue{Histogram: cur}
+		ret[i] = SequenceValue{Histogram: cur, CounterResetHintSet: counterResetHintSet}
 	}
 
 	return ret, nil
@@ -543,6 +558,8 @@ func (*parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uint6
 // buildHistogramFromMap is used in the grammar to take then individual parts of the histogram and complete it.
 func (p *parser) buildHistogramFromMap(desc *map[string]any) *histogram.FloatHistogram {
 	output := &histogram.FloatHistogram{}
+	// Reset the flag for each new histogram being built.
+	p.lastHistogramCounterResetHintSet = false
 
 	val, ok := (*desc)["schema"]
 	if ok {
@@ -603,6 +620,8 @@ func (p *parser) buildHistogramFromMap(desc *map[string]any) *histogram.FloatHis
 
 	val, ok = (*desc)["counter_reset_hint"]
 	if ok {
+		// Mark that the counter reset hint was explicitly specified.
+		p.lastHistogramCounterResetHintSet = true
 		resetHint, ok := val.(Item)
 
 		if ok {
@@ -634,6 +653,16 @@ func (p *parser) buildHistogramFromMap(desc *map[string]any) *histogram.FloatHis
 	return output
 }
 
+// newHistogramSequenceValue creates a SequenceValue for a histogram,
+// setting CounterResetHintSet based on whether counter_reset_hint was
+// explicitly specified in the histogram description.
+func (p *parser) newHistogramSequenceValue(h *histogram.FloatHistogram) SequenceValue {
+	return SequenceValue{
+		Histogram:           h,
+		CounterResetHintSet: p.lastHistogramCounterResetHintSet,
+	}
+}
+
 func (p *parser) buildHistogramBucketsAndSpans(desc *map[string]any, bucketsKey, offsetKey string,
 ) (buckets []float64, spans []histogram.Span) {
 	bucketCount := 0

promql/promql_test.go

@@ -39,7 +39,7 @@ func TestEvaluations(t *testing.T) {
 // Run a lot of queries at the same time, to check for race conditions.
 func TestConcurrentRangeQueries(t *testing.T) {
 	stor := teststorage.New(t)
-	defer stor.Close()
+
 	opts := promql.EngineOpts{
 		Logger:     nil,
 		Reg:        nil,

promql/promqltest/README.md

@@ -110,6 +110,15 @@ eval range from <start> to <end> step <step> <query>
 * `<expect string> "<string>"` (optional) for matching a string literal
 * `<series>` and `<points>` specify the expected values, and follow the same syntax as for `load` above
 
+### Special handling of counter reset hints in native histograms
+
+Native histograms as part of `<points>` may or may not contain an explicit
+`counter_reset_hint` property. If a `counter_reset_hint` is provided
+explicitly, the counter reset hint of the histogram is tested to have the
+provided value (`unknown`, `reset`, `not_reset`, or `gauge`). However, if no
+`counter_reset_hint` is specified, the `counter_reset_hint` is not tested at
+all (rather than testing for the usual default value `unknown`).
+
 ### `expect string`
 
 This can be used to specify that a string literal is the expected result.

promql/promqltest/test.go

@@ -1047,7 +1047,12 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 			exp := ev.expected[hash]
 
 			var expectedFloats []promql.FPoint
-			var expectedHistograms []promql.HPoint
+			// expectedHPoint wraps HPoint with CounterResetHintSet flag from SequenceValue.
+			type expectedHPoint struct {
+				promql.HPoint
+				CounterResetHintSet bool
+			}
+			var expectedHistograms []expectedHPoint
 
 			for i, e := range exp.vals {
 				ts := ev.start.Add(time.Duration(i) * ev.step)
@@ -1059,7 +1064,10 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 				t := ts.UnixNano() / int64(time.Millisecond/time.Nanosecond)
 
 				if e.Histogram != nil {
-					expectedHistograms = append(expectedHistograms, promql.HPoint{T: t, H: e.Histogram})
+					expectedHistograms = append(expectedHistograms, expectedHPoint{
+						HPoint:              promql.HPoint{T: t, H: e.Histogram},
+						CounterResetHintSet: e.CounterResetHintSet,
+					})
 				} else if !e.Omitted {
 					expectedFloats = append(expectedFloats, promql.FPoint{T: t, F: e.Value})
 				}
@@ -1088,7 +1096,7 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 					return fmt.Errorf("expected histogram value at index %v for %s to have timestamp %v, but it had timestamp %v (result has %s)", i, ev.metrics[hash], expected.T, actual.T, formatSeriesResult(s))
 				}
 
-				if !compareNativeHistogram(expected.H.Compact(0), actual.H.Compact(0)) {
+				if !compareNativeHistogram(expected.H.Compact(0), actual.H.Compact(0), expected.CounterResetHintSet) {
 					return fmt.Errorf("expected histogram value at index %v (t=%v) for %s to be %v, but got %v (result has %s)", i, actual.T, ev.metrics[hash], expected.H.TestExpression(), actual.H.TestExpression(), formatSeriesResult(s))
 				}
 			}
@@ -1127,7 +1135,7 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 			if expH != nil && v.H == nil {
 				return fmt.Errorf("expected histogram %s for %s but got float value %v", HistogramTestExpression(expH), v.Metric, v.F)
 			}
-			if expH != nil && !compareNativeHistogram(expH.Compact(0), v.H.Compact(0)) {
+			if expH != nil && !compareNativeHistogram(expH.Compact(0), v.H.Compact(0), exp0.CounterResetHintSet) {
 				return fmt.Errorf("expected %v for %s but got %s", HistogramTestExpression(expH), v.Metric, HistogramTestExpression(v.H))
 			}
 			if !almost.Equal(exp0.Value, v.F, defaultEpsilon) {
@@ -1165,7 +1173,9 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 
 // compareNativeHistogram is helper function to compare two native histograms
 // which can tolerate some differ in the field of float type, such as Count, Sum.
-func compareNativeHistogram(exp, cur *histogram.FloatHistogram) bool {
+// The counterResetHintSet parameter indicates whether the counter reset hint was
+// explicitly specified in the expected histogram (from the test file).
+func compareNativeHistogram(exp, cur *histogram.FloatHistogram, counterResetHintSet bool) bool {
 	if exp == nil || cur == nil {
 		return false
 	}
@@ -1201,6 +1211,15 @@ func compareNativeHistogram(exp, cur *histogram.FloatHistogram) bool {
 		return false
 	}
 
+	// Compare CounterResetHint only if explicitly specified in expected histogram.
+	// When counterResetHintSet is false, no hint was specified, meaning "don't care".
+	// When counterResetHintSet is true, the hint was explicitly specified and must match.
+	if counterResetHintSet {
+		if exp.CounterResetHint != cur.CounterResetHint {
+			return false
+		}
+	}
+
 	return true
 }
 

promql/promqltest/testdata/info.test

@@ -70,7 +70,11 @@ eval range from 0m to 10m step 5m info(metric, {__name__=~".+_info"})
     metric{instance="a", job="1", label="value", build_data="build", data="info", another_data="another info"} 0 1 2
 
 # Info metrics themselves are ignored when it comes to enriching with info metric data labels.
-eval range from 0m to 10m step 5m info(build_info, {__name__=~".+_info", build_data=~".+"})
+eval range from 0m to 10m step 5m info(build_info, {__name__=~".+_info", another_data=~".+"})
+    build_info{instance="a", job="1", build_data="build"} 1 1 1
+
+# Info metrics themselves are ignored when it comes to enriching with info metric data labels.
+eval range from 0m to 10m step 5m info(build_info, {__name__=~".+_info"})
     build_info{instance="a", job="1", build_data="build"} 1 1 1
 
 clear

promql/promqltest/testdata/native_histograms.test

@@ -1283,7 +1283,7 @@ eval instant at 12m sum_over_time(nhcb_metric[13m])
 eval instant at 12m avg_over_time(nhcb_metric[13m])
   expect no_warn
   expect info msg: PromQL info: mismatched custom buckets were reconciled during aggregation
-  {} {{schema:-53 count:1 sum:1 custom_values:[5] counter_reset_hint:gauge buckets:[1]}}
+  {} {{schema:-53 count:1 sum:1 custom_values:[5] buckets:[1]}}
 
 eval instant at 12m last_over_time(nhcb_metric[13m])
   expect no_warn

promql/query_logger.go

@@ -164,7 +164,7 @@ func trimStringByBytes(str string, size int) string {
 
 	trimIndex := len(bytesStr)
 	if size < len(bytesStr) {
-		for !utf8.RuneStart(bytesStr[size]) {
+		for size > 0 && !utf8.RuneStart(bytesStr[size]) {
 			size--
 		}
 		trimIndex = size

promql/query_logger_test.go

@@ -127,6 +127,47 @@ func TestMMapFile(t *testing.T) {
 	require.Equal(t, []byte(data), bytes[:2], "Mmap failed")
 }
 
+func TestTrimStringByBytes(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		input    string
+		size     int
+		expected string
+	}{
+		{
+			name:     "normal ASCII string",
+			input:    "hello",
+			size:     3,
+			expected: "hel",
+		},
+		{
+			name:     "no trimming needed",
+			input:    "hi",
+			size:     10,
+			expected: "hi",
+		},
+		{
+			name:     "UTF-8 multibyte character boundary",
+			input:    "日本", // 6 bytes (3 bytes per character)
+			size:     4,
+			expected: "日", // trims back to complete character boundary
+		},
+		{
+			name:     "invalid UTF-8 continuation-only bytes",
+			input:    string([]byte{0x80, 0x81, 0x82, 0x83, 0x84}), // only continuation bytes
+			size:     4,
+			expected: "",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			require.NotPanics(t, func() {
+				result := trimStringByBytes(tc.input, tc.size)
+				require.Equal(t, tc.expected, result)
+			})
+		})
+	}
+}
+
 func TestParseBrokenJSON(t *testing.T) {
 	for _, tc := range []struct {
 		b []byte

rules/alerting_test.go

@@ -158,7 +158,6 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
 		load 1m
 			http_requests{job="app-server", instance="0"}	75 85 70 70 stale
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	expr, err := parser.ParseExpr(`http_requests < 100`)
 	require.NoError(t, err)
@@ -264,7 +263,6 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
 		load 1m
 			http_requests{job="app-server", instance="0"}	75 85 70 70
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	expr, err := parser.ParseExpr(`http_requests < 100`)
 	require.NoError(t, err)
@@ -359,7 +357,6 @@ func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
 		load 1m
 			http_requests{job="app-server", instance="0"}	75 85 70 70
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	expr, err := parser.ParseExpr(`http_requests < 100`)
 	require.NoError(t, err)
@@ -454,7 +451,6 @@ func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
 		load 1m
 			http_requests{job="app-server", instance="0"}	75 85 70 70
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	expr, err := parser.ParseExpr(`http_requests < 100`)
 	require.NoError(t, err)
@@ -510,7 +506,6 @@ func TestAlertingRuleQueryInTemplate(t *testing.T) {
 		load 1m
 			http_requests{job="app-server", instance="0"}	70 85 70 70
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	expr, err := parser.ParseExpr(`sum(http_requests) < 100`)
 	require.NoError(t, err)
@@ -584,7 +579,6 @@ func BenchmarkAlertingRuleAtomicField(b *testing.B) {
 
 func TestAlertingRuleDuplicate(t *testing.T) {
 	storage := teststorage.New(t)
-	defer storage.Close()
 
 	opts := promql.EngineOpts{
 		Logger:     nil,
@@ -621,7 +615,6 @@ func TestAlertingRuleLimit(t *testing.T) {
 			metric{label="1"} 1
 			metric{label="2"} 1
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	tests := []struct {
 		limit int
@@ -805,7 +798,6 @@ func TestKeepFiringFor(t *testing.T) {
 		load 1m
 			http_requests{job="app-server", instance="0"}	75 85 70 70 10x5
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	expr, err := parser.ParseExpr(`http_requests > 50`)
 	require.NoError(t, err)
@@ -916,7 +908,6 @@ func TestPendingAndKeepFiringFor(t *testing.T) {
 		load 1m
 			http_requests{job="app-server", instance="0"}	75 10x10
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	expr, err := parser.ParseExpr(`http_requests > 50`)
 	require.NoError(t, err)

rules/manager_test.go

@@ -62,7 +62,6 @@ func TestAlertingRule(t *testing.T) {
 			http_requests{job="app-server", instance="0", group="canary", severity="overwrite-me"}	75 85  95 105 105  95  85
 			http_requests{job="app-server", instance="1", group="canary", severity="overwrite-me"}	80 90 100 110 120 130 140
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	expr, err := parser.ParseExpr(`http_requests{group="canary", job="app-server"} < 100`)
 	require.NoError(t, err)
@@ -205,7 +204,6 @@ func TestForStateAddSamples(t *testing.T) {
 			http_requests{job="app-server", instance="0", group="canary", severity="overwrite-me"}	75 85  95 105 105  95  85
 			http_requests{job="app-server", instance="1", group="canary", severity="overwrite-me"}	80 90 100 110 120 130 140
 	`)
-			t.Cleanup(func() { storage.Close() })
 
 			expr, err := parser.ParseExpr(`http_requests{group="canary", job="app-server"} < 100`)
 			require.NoError(t, err)
@@ -367,7 +365,6 @@ func TestForStateRestore(t *testing.T) {
 		http_requests{job="app-server", instance="0", group="canary", severity="overwrite-me"}	75  85 50 0 0 25 0 0 40 0 120
 		http_requests{job="app-server", instance="1", group="canary", severity="overwrite-me"}	125 90 60 0 0 25 0 0 40 0 130
 	`)
-			t.Cleanup(func() { storage.Close() })
 
 			expr, err := parser.ParseExpr(`http_requests{group="canary", job="app-server"} < 100`)
 			require.NoError(t, err)
@@ -538,7 +535,7 @@ func TestForStateRestore(t *testing.T) {
 func TestStaleness(t *testing.T) {
 	for _, queryOffset := range []time.Duration{0, time.Minute} {
 		st := teststorage.New(t)
-		defer st.Close()
+
 		engineOpts := promql.EngineOpts{
 			Logger:     nil,
 			Reg:        nil,
@@ -726,7 +723,7 @@ func TestCopyState(t *testing.T) {
 
 func TestDeletedRuleMarkedStale(t *testing.T) {
 	st := teststorage.New(t)
-	defer st.Close()
+
 	oldGroup := &Group{
 		rules: []Rule{
 			NewRecordingRule("rule1", nil, labels.FromStrings("l1", "v1")),
@@ -772,7 +769,7 @@ func TestUpdate(t *testing.T) {
 		"test": labels.FromStrings("name", "value"),
 	}
 	st := teststorage.New(t)
-	defer st.Close()
+
 	opts := promql.EngineOpts{
 		Logger:     nil,
 		Reg:        nil,
@@ -910,7 +907,7 @@ func reloadAndValidate(rgs *rulefmt.RuleGroups, t *testing.T, tmpFile *os.File,
 
 func TestNotify(t *testing.T) {
 	storage := teststorage.New(t)
-	defer storage.Close()
+
 	engineOpts := promql.EngineOpts{
 		Logger:     nil,
 		Reg:        nil,
@@ -984,7 +981,7 @@ func TestMetricsUpdate(t *testing.T) {
 	}
 
 	storage := teststorage.New(t)
-	defer storage.Close()
+
 	registry := prometheus.NewRegistry()
 	opts := promql.EngineOpts{
 		Logger:     nil,
@@ -1057,7 +1054,7 @@ func TestGroupStalenessOnRemoval(t *testing.T) {
 	sameFiles := []string{"fixtures/rules2_copy.yaml"}
 
 	storage := teststorage.New(t)
-	defer storage.Close()
+
 	opts := promql.EngineOpts{
 		Logger:     nil,
 		Reg:        nil,
@@ -1135,7 +1132,7 @@ func TestMetricsStalenessOnManagerShutdown(t *testing.T) {
 	files := []string{"fixtures/rules2.yaml"}
 
 	storage := teststorage.New(t)
-	defer storage.Close()
+
 	opts := promql.EngineOpts{
 		Logger:     nil,
 		Reg:        nil,
@@ -1205,7 +1202,7 @@ func TestRuleMovedBetweenGroups(t *testing.T) {
 	storage := teststorage.New(t, func(opt *tsdb.Options) {
 		opt.OutOfOrderTimeWindow = 600000
 	})
-	defer storage.Close()
+
 	opts := promql.EngineOpts{
 		Logger:     nil,
 		Reg:        nil,
@@ -1287,7 +1284,7 @@ func TestGroupHasAlertingRules(t *testing.T) {
 
 func TestRuleHealthUpdates(t *testing.T) {
 	st := teststorage.New(t)
-	defer st.Close()
+
 	engineOpts := promql.EngineOpts{
 		Logger:     nil,
 		Reg:        nil,
@@ -1348,7 +1345,6 @@ func TestRuleGroupEvalIterationFunc(t *testing.T) {
 		load 5m
 		http_requests{instance="0"}	75  85 50 0 0 25 0 0 40 0 120
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	expr, err := parser.ParseExpr(`http_requests{group="canary", job="app-server"} < 100`)
 	require.NoError(t, err)
@@ -1463,7 +1459,6 @@ func TestRuleGroupEvalIterationFunc(t *testing.T) {
 
 func TestNativeHistogramsInRecordingRules(t *testing.T) {
 	storage := teststorage.New(t)
-	t.Cleanup(func() { storage.Close() })
 
 	// Add some histograms.
 	db := storage.DB
@@ -1525,9 +1520,6 @@ func TestNativeHistogramsInRecordingRules(t *testing.T) {
 
 func TestManager_LoadGroups_ShouldCheckWhetherEachRuleHasDependentsAndDependencies(t *testing.T) {
 	storage := teststorage.New(t)
-	t.Cleanup(func() {
-		require.NoError(t, storage.Close())
-	})
 
 	ruleManager := NewManager(&ManagerOptions{
 		Context:    context.Background(),
@@ -2021,7 +2013,7 @@ func TestAsyncRuleEvaluation(t *testing.T) {
 	t.Run("synchronous evaluation with independent rules", func(t *testing.T) {
 		t.Parallel()
 		storage := teststorage.New(t)
-		t.Cleanup(func() { storage.Close() })
+
 		inflightQueries := atomic.Int32{}
 		maxInflight := atomic.Int32{}
 
@@ -2060,7 +2052,7 @@ func TestAsyncRuleEvaluation(t *testing.T) {
 	t.Run("asynchronous evaluation with independent and dependent rules", func(t *testing.T) {
 		t.Parallel()
 		storage := teststorage.New(t)
-		t.Cleanup(func() { storage.Close() })
+
 		inflightQueries := atomic.Int32{}
 		maxInflight := atomic.Int32{}
 
@@ -2099,7 +2091,7 @@ func TestAsyncRuleEvaluation(t *testing.T) {
 	t.Run("asynchronous evaluation of all independent rules, insufficient concurrency", func(t *testing.T) {
 		t.Parallel()
 		storage := teststorage.New(t)
-		t.Cleanup(func() { storage.Close() })
+
 		inflightQueries := atomic.Int32{}
 		maxInflight := atomic.Int32{}
 
@@ -2144,7 +2136,7 @@ func TestAsyncRuleEvaluation(t *testing.T) {
 	t.Run("asynchronous evaluation of all independent rules, sufficient concurrency", func(t *testing.T) {
 		t.Parallel()
 		storage := teststorage.New(t)
-		t.Cleanup(func() { storage.Close() })
+
 		inflightQueries := atomic.Int32{}
 		maxInflight := atomic.Int32{}
 
@@ -2192,7 +2184,7 @@ func TestAsyncRuleEvaluation(t *testing.T) {
 	t.Run("asynchronous evaluation of independent rules, with indeterminate. Should be synchronous", func(t *testing.T) {
 		t.Parallel()
 		storage := teststorage.New(t)
-		t.Cleanup(func() { storage.Close() })
+
 		inflightQueries := atomic.Int32{}
 		maxInflight := atomic.Int32{}
 
@@ -2231,7 +2223,7 @@ func TestAsyncRuleEvaluation(t *testing.T) {
 	t.Run("asynchronous evaluation of rules that benefit from reordering", func(t *testing.T) {
 		t.Parallel()
 		storage := teststorage.New(t)
-		t.Cleanup(func() { storage.Close() })
+
 		inflightQueries := atomic.Int32{}
 		maxInflight := atomic.Int32{}
 
@@ -2277,7 +2269,7 @@ func TestAsyncRuleEvaluation(t *testing.T) {
 	t.Run("attempted asynchronous evaluation of chained rules", func(t *testing.T) {
 		t.Parallel()
 		storage := teststorage.New(t)
-		t.Cleanup(func() { storage.Close() })
+
 		inflightQueries := atomic.Int32{}
 		maxInflight := atomic.Int32{}
 
@@ -2325,7 +2317,7 @@ func TestAsyncRuleEvaluation(t *testing.T) {
 func TestNewRuleGroupRestoration(t *testing.T) {
 	t.Parallel()
 	store := teststorage.New(t)
-	t.Cleanup(func() { store.Close() })
+
 	var (
 		inflightQueries atomic.Int32
 		maxInflight     atomic.Int32
@@ -2389,7 +2381,7 @@ func TestNewRuleGroupRestoration(t *testing.T) {
 func TestNewRuleGroupRestorationWithRestoreNewGroupOption(t *testing.T) {
 	t.Parallel()
 	store := teststorage.New(t)
-	t.Cleanup(func() { store.Close() })
+
 	var (
 		inflightQueries atomic.Int32
 		maxInflight     atomic.Int32
@@ -2459,7 +2451,6 @@ func TestNewRuleGroupRestorationWithRestoreNewGroupOption(t *testing.T) {
 
 func TestBoundedRuleEvalConcurrency(t *testing.T) {
 	storage := teststorage.New(t)
-	t.Cleanup(func() { storage.Close() })
 
 	var (
 		inflightQueries atomic.Int32
@@ -2514,7 +2505,6 @@ func TestUpdateWhenStopped(t *testing.T) {
 
 func TestGroup_Eval_RaceConditionOnStoppingGroupEvaluationWhileRulesAreEvaluatedConcurrently(t *testing.T) {
 	storage := teststorage.New(t)
-	t.Cleanup(func() { storage.Close() })
 
 	var (
 		inflightQueries atomic.Int32
@@ -2733,7 +2723,6 @@ func TestRuleDependencyController_AnalyseRules(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			storage := teststorage.New(t)
-			t.Cleanup(func() { storage.Close() })
 
 			ruleManager := NewManager(&ManagerOptions{
 				Context:    context.Background(),
@@ -2762,7 +2751,6 @@ func TestRuleDependencyController_AnalyseRules(t *testing.T) {
 
 func BenchmarkRuleDependencyController_AnalyseRules(b *testing.B) {
 	storage := teststorage.New(b)
-	b.Cleanup(func() { storage.Close() })
 
 	ruleManager := NewManager(&ManagerOptions{
 		Context:    context.Background(),

rules/recording_test.go

@@ -121,7 +121,6 @@ func setUpRuleEvalTest(t testing.TB) *teststorage.TestStorage {
 
 func TestRuleEval(t *testing.T) {
 	storage := setUpRuleEvalTest(t)
-	t.Cleanup(func() { storage.Close() })
 
 	ng := testEngine(t)
 	for _, scenario := range ruleEvalTestScenarios {
@@ -158,7 +157,6 @@ func BenchmarkRuleEval(b *testing.B) {
 // TestRuleEvalDuplicate tests for duplicate labels in recorded metrics, see #5529.
 func TestRuleEvalDuplicate(t *testing.T) {
 	storage := teststorage.New(t)
-	defer storage.Close()
 
 	opts := promql.EngineOpts{
 		Logger:     nil,
@@ -185,7 +183,6 @@ func TestRecordingRuleLimit(t *testing.T) {
 			metric{label="1"} 1
 			metric{label="2"} 1
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	tests := []struct {
 		limit int

scrape/manager.go

@@ -39,14 +39,35 @@ import (
 	"github.com/prometheus/prometheus/util/pool"
 )
 
-// NewManager is the Manager constructor using Appendable.
-func NewManager(o *Options, logger *slog.Logger, newScrapeFailureLogger func(string) (*logging.JSONFileLogger, error), appendable storage.Appendable, registerer prometheus.Registerer) (*Manager, error) {
+// NewManager is the Manager constructor using storage.Appendable or storage.AppendableV2.
+//
+// If unsure which one to use/implement, implement AppendableV2 as it significantly simplifies implementation and allows more
+// (passing ST, always-on metadata, exemplars per sample).
+//
+// NewManager returns error if both appendable and appendableV2 are specified.
+//
+// Switch to AppendableV2 is in progress (https://github.com/prometheus/prometheus/issues/17632).
+// storage.Appendable will be removed soon (ETA: Q2 2026).
+func NewManager(
+	o *Options,
+	logger *slog.Logger,
+	newScrapeFailureLogger func(string) (*logging.JSONFileLogger, error),
+	appendable storage.Appendable,
+	appendableV2 storage.AppendableV2,
+	registerer prometheus.Registerer,
+) (*Manager, error) {
 	if o == nil {
 		o = &Options{}
 	}
 	if logger == nil {
 		logger = promslog.NewNopLogger()
 	}
+	if appendable != nil && appendableV2 != nil {
+		return nil, errors.New("scrape.NewManager: appendable and appendableV2 cannot be provided at the same time")
+	}
+	if appendable == nil && appendableV2 == nil {
+		return nil, errors.New("scrape.NewManager: provide either appendable or appendableV2")
+	}
 
 	sm, err := newScrapeMetrics(registerer)
 	if err != nil {
@@ -55,6 +76,7 @@ func NewManager(o *Options, logger *slog.Logger, newScrapeFailureLogger func(str
 
 	m := &Manager{
 		appendable:             appendable,
+		appendableV2:           appendableV2,
 		opts:                   o,
 		logger:                 logger,
 		newScrapeFailureLogger: newScrapeFailureLogger,

scrape/manager_test.go

@@ -522,7 +522,7 @@ scrape_configs:
 	)
 
 	opts := Options{}
-	scrapeManager, err := NewManager(&opts, nil, nil, nil, testRegistry)
+	scrapeManager, err := NewManager(&opts, nil, nil, nil, teststorage.NewAppendable(), testRegistry)
 	require.NoError(t, err)
 	newLoop := func(scrapeLoopOptions) loop {
 		ch <- struct{}{}
@@ -578,11 +578,11 @@ scrape_configs:
 func TestManagerTargetsUpdates(t *testing.T) {
 	opts := Options{}
 	testRegistry := prometheus.NewRegistry()
-	m, err := NewManager(&opts, nil, nil, nil, testRegistry)
+	m, err := NewManager(&opts, nil, nil, nil, teststorage.NewAppendable(), testRegistry)
 	require.NoError(t, err)
 
-	ts := make(chan map[string][]*targetgroup.Group)
-	go m.Run(ts)
+	targetSetsCh := make(chan map[string][]*targetgroup.Group)
+	go m.Run(targetSetsCh)
 	defer m.Stop()
 
 	tgSent := make(map[string][]*targetgroup.Group)
@@ -594,7 +594,7 @@ func TestManagerTargetsUpdates(t *testing.T) {
 		}
 
 		select {
-		case ts <- tgSent:
+		case targetSetsCh <- tgSent:
 		case <-time.After(10 * time.Millisecond):
 			require.Fail(t, "Scrape manager's channel remained blocked after the set threshold.")
 		}
@@ -631,7 +631,7 @@ global:
 
 	opts := Options{}
 	testRegistry := prometheus.NewRegistry()
-	scrapeManager, err := NewManager(&opts, nil, nil, nil, testRegistry)
+	scrapeManager, err := NewManager(&opts, nil, nil, nil, teststorage.NewAppendable(), testRegistry)
 	require.NoError(t, err)
 
 	// Load the first config.
@@ -701,7 +701,7 @@ scrape_configs:
 	}
 
 	opts := Options{}
-	scrapeManager, err := NewManager(&opts, nil, nil, nil, testRegistry)
+	scrapeManager, err := NewManager(&opts, nil, nil, nil, teststorage.NewAppendable(), testRegistry)
 	require.NoError(t, err)
 
 	reload(scrapeManager, cfg1)
@@ -735,6 +735,8 @@ func setupTestServer(t *testing.T, typ string, toWrite []byte) *httptest.Server
 }
 
 // TestManagerSTZeroIngestion tests scrape manager for various ST cases.
+// NOTE(bwplotka): There is no AppenderV2 test for this STZeroIngestion feature as in V2 flow it's
+// moved to AppenderV2 implementation (e.g. storage) and it's tested there, e.g. tsdb.TestHeadAppenderV2_Append_EnableSTAsZeroSample.
 func TestManagerSTZeroIngestion(t *testing.T) {
 	t.Parallel()
 	const (
@@ -766,7 +768,7 @@ func TestManagerSTZeroIngestion(t *testing.T) {
 							discoveryManager, scrapeManager := runManagers(t, ctx, &Options{
 								EnableStartTimestampZeroIngestion: testSTZeroIngest,
 								skipOffsetting:                    true,
-							}, app)
+							}, app, nil)
 							defer scrapeManager.Stop()
 
 							server := setupTestServer(t, config.ScrapeProtocolsHeaders[testFormat], encoded)
@@ -905,6 +907,8 @@ func generateTestHistogram(i int) *dto.Histogram {
 	return h
 }
 
+// NOTE(bwplotka): There is no AppenderV2 test for this STZeroIngestion feature as in V2 flow it's
+// moved to AppenderV2 implementation (e.g. storage) and it's tested there, e.g. tsdb.TestHeadAppenderV2_Append_EnableSTAsZeroSample.
 func TestManagerSTZeroIngestionHistogram(t *testing.T) {
 	t.Parallel()
 	const mName = "expected_histogram"
@@ -950,7 +954,7 @@ func TestManagerSTZeroIngestionHistogram(t *testing.T) {
 			discoveryManager, scrapeManager := runManagers(t, ctx, &Options{
 				EnableStartTimestampZeroIngestion: tc.enableSTZeroIngestion,
 				skipOffsetting:                    true,
-			}, app)
+			}, app, nil)
 			defer scrapeManager.Stop()
 
 			once := sync.Once{}
@@ -1030,7 +1034,7 @@ func TestUnregisterMetrics(t *testing.T) {
 	// Check that all metrics can be unregistered, allowing a second manager to be created.
 	for range 2 {
 		opts := Options{}
-		manager, err := NewManager(&opts, nil, nil, nil, reg)
+		manager, err := NewManager(&opts, nil, nil, nil, teststorage.NewAppendable(), reg)
 		require.NotNil(t, manager)
 		require.NoError(t, err)
 		// Unregister all metrics.
@@ -1043,6 +1047,9 @@ func TestUnregisterMetrics(t *testing.T) {
 // This test addresses issue #17216 by ensuring the previously blocking check has been removed.
 // The test verifies that the presence of exemplars in the input does not cause errors,
 // although exemplars are not preserved during NHCB conversion (as documented below).
+//
+// NOTE(bwplotka): There is no AppenderV2 test for this STZeroIngestion feature as in V2 flow it's
+// moved to AppenderV2 implementation (e.g. storage) and it's tested there, e.g. tsdb.TestHeadAppenderV2_Append_EnableSTAsZeroSample.
 func TestNHCBAndSTZeroIngestion(t *testing.T) {
 	t.Parallel()
 
@@ -1059,7 +1066,7 @@ func TestNHCBAndSTZeroIngestion(t *testing.T) {
 	discoveryManager, scrapeManager := runManagers(t, ctx, &Options{
 		EnableStartTimestampZeroIngestion: true,
 		skipOffsetting:                    true,
-	}, app)
+	}, app, nil)
 	defer scrapeManager.Stop()
 
 	once := sync.Once{}
@@ -1153,16 +1160,13 @@ func applyConfig(
 	require.NoError(t, discoveryManager.ApplyConfig(c))
 }
 
-func runManagers(t *testing.T, ctx context.Context, opts *Options, app storage.Appendable) (*discovery.Manager, *Manager) {
+func runManagers(t *testing.T, ctx context.Context, opts *Options, app storage.Appendable, appV2 storage.AppendableV2) (*discovery.Manager, *Manager) {
 	t.Helper()
 
 	if opts == nil {
 		opts = &Options{}
 	}
 	opts.DiscoveryReloadInterval = model.Duration(100 * time.Millisecond)
-	if app == nil {
-		app = teststorage.NewAppendable()
-	}
 
 	reg := prometheus.NewRegistry()
 	sdMetrics, err := discovery.RegisterSDMetrics(reg, discovery.NewRefreshMetrics(reg))
@@ -1178,7 +1182,7 @@ func runManagers(t *testing.T, ctx context.Context, opts *Options, app storage.A
 		opts,
 		nil,
 		nil,
-		app,
+		app, appV2,
 		prometheus.NewRegistry(),
 	)
 	require.NoError(t, err)
@@ -1251,7 +1255,7 @@ scrape_configs:
   - files: ['%s']
 `
 
-	discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil)
+	discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil, teststorage.NewAppendable())
 	defer scrapeManager.Stop()
 
 	applyConfig(
@@ -1350,7 +1354,7 @@ scrape_configs:
   file_sd_configs:
   - files: ['%s', '%s']
 `
-	discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil)
+	discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil, teststorage.NewAppendable())
 	defer scrapeManager.Stop()
 
 	applyConfig(
@@ -1409,7 +1413,7 @@ scrape_configs:
   file_sd_configs:
   - files: ['%s']
 `
-	discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil)
+	discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil, teststorage.NewAppendable())
 	defer scrapeManager.Stop()
 
 	applyConfig(
@@ -1475,7 +1479,7 @@ scrape_configs:
   - targets: ['%s']
 `
 
-	discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil)
+	discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil, teststorage.NewAppendable())
 	defer scrapeManager.Stop()
 
 	// Apply the initial config with an existing file
@@ -1559,7 +1563,7 @@ scrape_configs:
 
 	cfg := loadConfiguration(t, cfgText)
 
-	m, err := NewManager(&Options{}, nil, nil, teststorage.NewAppendable(), prometheus.NewRegistry())
+	m, err := NewManager(&Options{}, nil, nil, nil, teststorage.NewAppendable(), prometheus.NewRegistry())
 	require.NoError(t, err)
 	defer m.Stop()
 	require.NoError(t, m.ApplyConfig(cfg))

scrape/scrape_test.go

@@ -131,7 +131,6 @@ func testStorageHandlesOutOfOrderTimestamps(t *testing.T, appV2 bool) {
 	// Test with default OutOfOrderTimeWindow (0)
 	t.Run("Out-Of-Order Sample Disabled", func(t *testing.T) {
 		s := teststorage.New(t)
-		t.Cleanup(func() { _ = s.Close() })
 		runScrapeLoopTest(t, appV2, s, false)
 	})
 
@@ -140,7 +139,6 @@ func testStorageHandlesOutOfOrderTimestamps(t *testing.T, appV2 bool) {
 		s := teststorage.New(t, func(opt *tsdb.Options) {
 			opt.OutOfOrderTimeWindow = 600000
 		})
-		t.Cleanup(func() { _ = s.Close() })
 
 		runScrapeLoopTest(t, appV2, s, true)
 	})
@@ -1438,7 +1436,9 @@ func readTextParseTestMetrics(t testing.TB) []byte {
 	if err != nil {
 		t.Fatal(err)
 	}
-	return b
+
+	// Replace all Carriage Return chars that appear when testing on windows.
+	return bytes.ReplaceAll(b, []byte{'\r'}, nil)
 }
 
 func makeTestGauges(n int) []byte {
@@ -1545,6 +1545,184 @@ func TestPromTextToProto(t *testing.T) {
 	require.Equal(t, "promhttp_metric_handler_requests_total", got[236])
 }
 
+// TestScrapeLoopAppend_WithStorage tests appends and storage integration for the
+// large input files that are also used in benchmarks.
+func TestScrapeLoopAppend_WithStorage(t *testing.T) {
+	ts := time.Now()
+
+	for _, appV2 := range []bool{false, true} {
+		for _, tc := range []struct {
+			name         string
+			parsableText []byte
+
+			expectedSamplesLen  int
+			testAppendedSamples func(t *testing.T, committed []sample)
+			testExemplars       func(t *testing.T, er []exemplar.QueryResult)
+		}{
+			{
+				name:         "1Fam2000Gauges",
+				parsableText: makeTestGauges(2000),
+
+				expectedSamplesLen: 2000,
+				testAppendedSamples: func(t *testing.T, committed []sample) {
+					var expectedMF string
+					if appV2 {
+						expectedMF = "metric_a" // Only AppenderV2 supports metric family passing.
+					}
+					// Verify a few samples.
+					testutil.RequireEqual(t, sample{
+						MF: expectedMF,
+						M:  metadata.Metadata{Type: model.MetricTypeGauge, Help: "help text"},
+						L:  labels.FromStrings(model.MetricNameLabel, "metric_a", "foo", "0", "bar", "0"), V: 1, T: timestamp.FromTime(ts),
+					}, committed[0])
+					testutil.RequireEqual(t, sample{
+						MF: expectedMF,
+						M:  metadata.Metadata{Type: model.MetricTypeGauge, Help: "help text"},
+						L:  labels.FromStrings(model.MetricNameLabel, "metric_a", "foo", "1245", "bar", "124500"), V: 1, T: timestamp.FromTime(ts),
+					}, committed[1245])
+					testutil.RequireEqual(t, sample{
+						MF: expectedMF,
+						M:  metadata.Metadata{Type: model.MetricTypeGauge, Help: "help text"},
+						L:  labels.FromStrings(model.MetricNameLabel, "metric_a", "foo", "1999", "bar", "199900"), V: 1, T: timestamp.FromTime(ts),
+					}, committed[len(committed)-1])
+				},
+			},
+			{
+				name:         "237FamsAllTypes",
+				parsableText: readTextParseTestMetrics(t),
+
+				expectedSamplesLen: 1857,
+				testAppendedSamples: func(t *testing.T, committed []sample) {
+					// Verify a few samples.
+					testutil.RequireEqual(t, sample{
+						MF: func() string {
+							if !appV2 {
+								return ""
+							}
+							return "go_gc_gomemlimit_bytes"
+						}(),
+						M: metadata.Metadata{Type: model.MetricTypeGauge, Help: "Go runtime memory limit configured by the user, otherwise math.MaxInt64. This value is set by the GOMEMLIMIT environment variable, and the runtime/debug.SetMemoryLimit function. Sourced from /gc/gomemlimit:bytes"},
+						L: labels.FromStrings(model.MetricNameLabel, "go_gc_gomemlimit_bytes"), V: 9.03676723e+08, T: timestamp.FromTime(ts),
+					}, committed[11])
+					testutil.RequireEqual(t, sample{
+						MF: func() string {
+							if !appV2 {
+								return "" // Only AppenderV2 supports metric family passing.
+							}
+							return "prometheus_http_request_duration_seconds"
+						}(),
+						M: metadata.Metadata{Type: model.MetricTypeHistogram, Help: "Histogram of latencies for HTTP requests."},
+						L: labels.FromStrings(model.MetricNameLabel, "prometheus_http_request_duration_seconds_bucket", "handler", "/api/v1/query_range", "le", "120.0"), V: 118157, T: timestamp.FromTime(ts),
+					}, committed[448])
+					testutil.RequireEqual(t, sample{
+						MF: func() string {
+							if !appV2 {
+								return "" // Only AppenderV2 supports metric family passing.
+							}
+							return "promhttp_metric_handler_requests_total"
+						}(),
+						M: metadata.Metadata{Type: model.MetricTypeCounter, Help: "Total number of scrapes by HTTP status code."},
+						L: labels.FromStrings(model.MetricNameLabel, "promhttp_metric_handler_requests_total", "code", "503"), V: 0, T: timestamp.FromTime(ts),
+					}, committed[len(committed)-1])
+				},
+			},
+			{
+				name:         "100HistsWithExemplars",
+				parsableText: makeTestHistogramsWithExemplars(100),
+
+				expectedSamplesLen: 24 * 100,
+				testAppendedSamples: func(t *testing.T, committed []sample) {
+					// Verify a few samples.
+					m := metadata.Metadata{Type: model.MetricTypeHistogram, Help: "RPC latency distributions."}
+					testutil.RequireEqual(t, sample{
+						MF: func() string {
+							if !appV2 {
+								return "" // Only AppenderV2 supports metric family passing.
+							}
+							return "rpc_durations_histogram0_seconds"
+						}(),
+						M: m, L: labels.FromStrings(model.MetricNameLabel, "rpc_durations_histogram0_seconds_bucket", "le", "0.0003100000000000002"), V: 15, T: timestamp.FromTime(ts),
+						ES: []exemplar.Exemplar{
+							{Labels: labels.FromStrings("dummyID", "9818"), Value: 0.0002791130914009552, Ts: 1726839814982, HasTs: true},
+						},
+					}, committed[13])
+					testutil.RequireEqual(t, sample{
+						MF: func() string {
+							if !appV2 {
+								return "" // Only AppenderV2 supports metric family passing.
+							}
+							return "rpc_durations_histogram49_seconds"
+						}(),
+						M: m, L: labels.FromStrings(model.MetricNameLabel, "rpc_durations_histogram49_seconds_sum"), V: -8.452185437166741e-05, T: timestamp.FromTime(ts),
+					}, committed[24*50-3])
+
+					// This series does not have metadata, nor metric family, because of isSeriesPartOfFamily bug and OpenMetric 1.0 limitations around _created series.
+					// TODO(bwplotka): Fix with https://github.com/prometheus/prometheus/issues/17900
+					testutil.RequireEqual(t, sample{
+						L: labels.FromStrings(model.MetricNameLabel, "rpc_durations_histogram99_seconds_created"), V: 1.726839813016302e+09, T: timestamp.FromTime(ts),
+					}, committed[len(committed)-1])
+				},
+				testExemplars: func(t *testing.T, er []exemplar.QueryResult) {
+					// 12 out of 24 histogram series have exemplars.
+					require.Len(t, er, 12*100)
+					testutil.RequireEqual(t, exemplar.QueryResult{
+						SeriesLabels: labels.FromStrings(model.MetricNameLabel, "rpc_durations_histogram0_seconds_bucket", "le", "0.0003100000000000002"),
+						Exemplars: []exemplar.Exemplar{
+							{Labels: labels.FromStrings("dummyID", "9818"), Value: 0.0002791130914009552, Ts: 1726839814982, HasTs: true},
+						},
+					}, er[10])
+					testutil.RequireEqual(t, exemplar.QueryResult{
+						SeriesLabels: labels.FromStrings(model.MetricNameLabel, "rpc_durations_histogram9_seconds_bucket", "le", "1.0000000000000216e-05"),
+						Exemplars: []exemplar.Exemplar{
+							{Labels: labels.FromStrings("dummyID", "19206"), Value: -4.6156147425468016e-05, Ts: 1726839815133, HasTs: true},
+						},
+					}, er[len(er)-1])
+				},
+			},
+		} {
+			t.Run(fmt.Sprintf("appV2=%v/data=%v", appV2, tc.name), func(t *testing.T) {
+				s := teststorage.New(t, func(opt *tsdb.Options) {
+					opt.EnableMetadataWALRecords = true
+				})
+
+				appTest := teststorage.NewAppendable().Then(s)
+				sl, _ := newTestScrapeLoop(t, withAppendable(appTest, appV2))
+				app := sl.appender()
+
+				_, _, _, err := app.append(tc.parsableText, "application/openmetrics-text", ts)
+				require.NoError(t, err)
+				require.NoError(t, app.Commit())
+
+				// Check the recorded samples on the Appender layer.
+				require.Nil(t, appTest.PendingSamples())
+				require.Nil(t, appTest.RolledbackSamples())
+
+				got := appTest.ResultSamples()
+				require.Len(t, got, tc.expectedSamplesLen)
+				tc.testAppendedSamples(t, got)
+
+				// Check basic storage stats.
+				stats := s.Head().Stats(model.MetricNameLabel, 2000)
+				require.Equal(t, tc.expectedSamplesLen, int(stats.NumSeries))
+
+				// Check exemplars.
+				eq, err := s.ExemplarQuerier(t.Context())
+				require.NoError(t, err)
+
+				er, err := eq.Select(math.MinInt64, math.MaxInt64, nil)
+				require.NoError(t, err)
+
+				if tc.testExemplars != nil {
+					tc.testExemplars(t, er)
+				} else {
+					// Expect no exemplars.
+					require.Empty(t, er, "%v is not empty", er)
+				}
+			})
+		}
+	}
+}
+
 // BenchmarkScrapeLoopAppend benchmarks scrape appends for typical cases.
 //
 // Benchmark compares append function run across 4 dimensions:
@@ -1569,7 +1747,7 @@ func BenchmarkScrapeLoopAppend(b *testing.B) {
 				name         string
 				parsableText []byte
 			}{
-				{name: "1Fam1000Gauges", parsableText: makeTestGauges(2000)},         // ~68.1 KB, ~77.9 KB in proto.
+				{name: "1Fam2000Gauges", parsableText: makeTestGauges(2000)},         // ~68.1 KB, ~77.9 KB in proto.
 				{name: "237FamsAllTypes", parsableText: readTextParseTestMetrics(b)}, // ~185.7 KB, ~70.6 KB in proto.
 			} {
 				b.Run(fmt.Sprintf("appV2=%v/appendMetadataToWAL=%v/data=%v", appV2, appendMetadataToWAL, data.name), func(b *testing.B) {
@@ -1610,7 +1788,6 @@ func benchScrapeLoopAppend(
 			opt.MaxExemplars = 1e5
 		}
 	})
-	b.Cleanup(func() { _ = s.Close() })
 
 	sl, _ := newTestScrapeLoop(b, withAppendable(s, appV2), func(sl *scrapeLoop) {
 		sl.appendMetadataToWAL = appendMetadataToWAL
@@ -1697,7 +1874,6 @@ func BenchmarkScrapeLoopScrapeAndReport(b *testing.B) {
 			parsableText := readTextParseTestMetrics(b)
 
 			s := teststorage.New(b)
-			b.Cleanup(func() { _ = s.Close() })
 
 			sl, scraper := newTestScrapeLoop(b, withAppendable(s, appV2), func(sl *scrapeLoop) {
 				sl.fallbackScrapeProtocol = "application/openmetrics-text"
@@ -1730,7 +1906,6 @@ func testSetOptionsHandlingStaleness(t *testing.T, appV2 bool) {
 	s := teststorage.New(t, func(opt *tsdb.Options) {
 		opt.OutOfOrderTimeWindow = 600000
 	})
-	t.Cleanup(func() { _ = s.Close() })
 
 	signal := make(chan struct{}, 1)
 	ctx, cancel := context.WithCancel(t.Context())
@@ -2001,7 +2176,6 @@ func TestScrapeLoopCache(t *testing.T) {
 
 func testScrapeLoopCache(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	signal := make(chan struct{}, 1)
 
@@ -2071,7 +2245,6 @@ func TestScrapeLoopCacheMemoryExhaustionProtection(t *testing.T) {
 
 func testScrapeLoopCacheMemoryExhaustionProtection(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	signal := make(chan struct{}, 1)
 
@@ -3225,9 +3398,7 @@ metric: <
 				}
 				sl.alwaysScrapeClassicHist = test.alwaysScrapeClassicHist
 				// This test does not care about metadata.
-				// Having this true would mean we need to add metadata to sample
-				// expectations.
-				// TODO(bwplotka): Add cases for append metadata to WAL and pass metadata
+				// TODO(bwplotka): Add metadata expectations and turn it on.
 				sl.appendMetadataToWAL = false
 			})
 			app := sl.appender()
@@ -3881,7 +4052,6 @@ func TestScrapeLoop_RespectTimestamps(t *testing.T) {
 
 func testScrapeLoopRespectTimestamps(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	appTest := teststorage.NewAppendable().Then(s)
 	sl, _ := newTestScrapeLoop(t, withAppendable(appTest, appV2))
@@ -3910,7 +4080,6 @@ func TestScrapeLoop_DiscardTimestamps(t *testing.T) {
 
 func testScrapeLoopDiscardTimestamps(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	appTest := teststorage.NewAppendable().Then(s)
 	sl, _ := newTestScrapeLoop(t, withAppendable(appTest, appV2), func(sl *scrapeLoop) {
@@ -3941,7 +4110,6 @@ func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) {
 
 func testScrapeLoopDiscardDuplicateLabels(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	appTest := teststorage.NewAppendable().Then(s)
 	sl, _ := newTestScrapeLoop(t, withAppendable(appTest, appV2))
@@ -3983,7 +4151,6 @@ func TestScrapeLoopDiscardUnnamedMetrics(t *testing.T) {
 
 func testScrapeLoopDiscardUnnamedMetrics(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	appTest := teststorage.NewAppendable().Then(s)
 	sl, _ := newTestScrapeLoop(t, withAppendable(appTest, appV2), func(sl *scrapeLoop) {
@@ -4274,7 +4441,6 @@ func TestScrapeAddFast(t *testing.T) {
 
 func testScrapeAddFast(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	sl, _ := newTestScrapeLoop(t, withAppendable(s, appV2))
 
@@ -4357,7 +4523,6 @@ func TestScrapeReportSingleAppender(t *testing.T) {
 func testScrapeReportSingleAppender(t *testing.T, appV2 bool) {
 	t.Parallel()
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	signal := make(chan struct{}, 1)
 
@@ -4417,7 +4582,6 @@ func TestScrapeReportLimit(t *testing.T) {
 
 func testScrapeReportLimit(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	cfg := &config.ScrapeConfig{
 		JobName:                    "test",
@@ -4480,7 +4644,6 @@ func TestScrapeUTF8(t *testing.T) {
 
 func testScrapeUTF8(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	cfg := &config.ScrapeConfig{
 		JobName:                    "test",
@@ -4678,7 +4841,6 @@ func TestLeQuantileReLabel(t *testing.T) {
 
 func testLeQuantileReLabel(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	cfg := &config.ScrapeConfig{
 		JobName: "test",
@@ -5205,7 +5367,6 @@ metric: <
 			t.Run(fmt.Sprintf("%s with %s", name, metricsTextName), func(t *testing.T) {
 				t.Parallel()
 				s := teststorage.New(t)
-				t.Cleanup(func() { _ = s.Close() })
 
 				sl, _ := newTestScrapeLoop(t, withAppendable(s, appV2), func(sl *scrapeLoop) {
 					sl.alwaysScrapeClassicHist = tc.alwaysScrapeClassicHistograms
@@ -5293,7 +5454,6 @@ func TestTypeUnitReLabel(t *testing.T) {
 
 func testTypeUnitReLabel(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	cfg := &config.ScrapeConfig{
 		JobName: "test",
@@ -5438,7 +5598,6 @@ func TestScrapeLoopCompression(t *testing.T) {
 
 func testScrapeLoopCompression(t *testing.T, appV2 bool) {
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 
 	metricsText := makeTestGauges(10)
 
@@ -5768,17 +5927,12 @@ scrape_configs:
 `, minBucketFactor, strings.ReplaceAll(metricsServer.URL, "http://", ""))
 
 	s := teststorage.New(t)
-	t.Cleanup(func() { _ = s.Close() })
 	reg := prometheus.NewRegistry()
 
-	mng, err := NewManager(&Options{DiscoveryReloadInterval: model.Duration(10 * time.Millisecond)}, nil, nil, s, reg)
+	sa := selectAppendable(s, appV2)
+	mng, err := NewManager(&Options{DiscoveryReloadInterval: model.Duration(10 * time.Millisecond)}, nil, nil, sa.V1(), sa.V2(), reg)
 	require.NoError(t, err)
 
-	if appV2 {
-		mng.appendableV2 = s
-		mng.appendable = nil
-	}
-
 	cfg, err := config.Load(configStr, promslog.NewNopLogger())
 	require.NoError(t, err)
 	require.NoError(t, mng.ApplyConfig(cfg))
@@ -6464,7 +6618,6 @@ func testNewScrapeLoopHonorLabelsWiring(t *testing.T, appV2 bool) {
 			require.NoError(t, err)
 
 			s := teststorage.New(t)
-			defer s.Close()
 
 			cfg := &config.ScrapeConfig{
 				JobName:                    "test",

storage/fanout_test.go

@@ -39,7 +39,6 @@ func TestFanout_SelectSorted(t *testing.T) {
 	ctx := context.Background()
 
 	priStorage := teststorage.New(t)
-	defer priStorage.Close()
 	app1 := priStorage.Appender(ctx)
 	app1.Append(0, inputLabel, 0, 0)
 	inputTotalSize++
@@ -51,7 +50,6 @@ func TestFanout_SelectSorted(t *testing.T) {
 	require.NoError(t, err)
 
 	remoteStorage1 := teststorage.New(t)
-	defer remoteStorage1.Close()
 	app2 := remoteStorage1.Appender(ctx)
 	app2.Append(0, inputLabel, 3000, 3)
 	inputTotalSize++
@@ -63,7 +61,6 @@ func TestFanout_SelectSorted(t *testing.T) {
 	require.NoError(t, err)
 
 	remoteStorage2 := teststorage.New(t)
-	defer remoteStorage2.Close()
 
 	app3 := remoteStorage2.Appender(ctx)
 	app3.Append(0, inputLabel, 6000, 6)
@@ -142,7 +139,6 @@ func TestFanout_SelectSorted_AppenderV2(t *testing.T) {
 	inputTotalSize := 0
 
 	priStorage := teststorage.New(t)
-	defer priStorage.Close()
 	app1 := priStorage.AppenderV2(t.Context())
 	_, err := app1.Append(0, inputLabel, 0, 0, 0, nil, nil, storage.AOptions{})
 	require.NoError(t, err)
@@ -156,7 +152,6 @@ func TestFanout_SelectSorted_AppenderV2(t *testing.T) {
 	require.NoError(t, app1.Commit())
 
 	remoteStorage1 := teststorage.New(t)
-	defer remoteStorage1.Close()
 	app2 := remoteStorage1.AppenderV2(t.Context())
 	_, err = app2.Append(0, inputLabel, 0, 3000, 3, nil, nil, storage.AOptions{})
 	require.NoError(t, err)
@@ -170,8 +165,6 @@ func TestFanout_SelectSorted_AppenderV2(t *testing.T) {
 	require.NoError(t, app2.Commit())
 
 	remoteStorage2 := teststorage.New(t)
-	defer remoteStorage2.Close()
-
 	app3 := remoteStorage2.AppenderV2(t.Context())
 	_, err = app3.Append(0, inputLabel, 0, 6000, 6, nil, nil, storage.AOptions{})
 	require.NoError(t, err)
@@ -246,7 +239,6 @@ func TestFanout_SelectSorted_AppenderV2(t *testing.T) {
 
 func TestFanoutErrors(t *testing.T) {
 	workingStorage := teststorage.New(t)
-	defer workingStorage.Close()
 
 	cases := []struct {
 		primary   storage.Storage

storage/remote/otlptranslator/prometheusremotewrite/helper.go

@@ -61,6 +61,13 @@ const (
 	defaultLookbackDelta = 5 * time.Minute
 )
 
+// reservedLabelNames contains label names that should be filtered from
+// OTLP attributes because they are set separately (via extras parameter).
+// Allowing these through could create duplicate labels.
+var reservedLabelNames = []string{
+	model.MetricNameLabel, // "__name__" - set from metric name
+}
+
 // createAttributes creates a slice of Prometheus Labels with OTLP attributes and pairs of string values.
 // Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen and
 // if logOnOverwrite is true, the overwrite is logged. Resulting label names are sanitized.
@@ -214,7 +221,7 @@ func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPo
 		pt := dataPoints.At(x)
 		timestamp := convertTimeStamp(pt.Timestamp())
 		startTimestamp := convertTimeStamp(pt.StartTimestamp())
-		baseLabels, err := c.createAttributes(pt.Attributes(), settings, nil, false, meta)
+		baseLabels, err := c.createAttributes(pt.Attributes(), settings, reservedLabelNames, false, meta)
 		if err != nil {
 			return err
 		}
@@ -416,7 +423,7 @@ func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoin
 		pt := dataPoints.At(x)
 		timestamp := convertTimeStamp(pt.Timestamp())
 		startTimestamp := convertTimeStamp(pt.StartTimestamp())
-		baseLabels, err := c.createAttributes(pt.Attributes(), settings, nil, false, meta)
+		baseLabels, err := c.createAttributes(pt.Attributes(), settings, reservedLabelNames, false, meta)
 		if err != nil {
 			return err
 		}

storage/remote/otlptranslator/prometheusremotewrite/helper_test.go

@@ -31,11 +31,12 @@ import (
 
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/metadata"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/util/testutil"
 )
 
-func TestCreateAttributes(t *testing.T) {
+func TestPrometheusConverter_createAttributes(t *testing.T) {
 	resourceAttrs := map[string]string{
 		"service.name":        "service name",
 		"service.instance.id": "service ID",
@@ -386,6 +387,18 @@ func TestCreateAttributes(t *testing.T) {
 				"metric_multi", "multi metric",
 			),
 		},
+		{
+			name:                      "__name__ attribute is filtered when passed in ignoreAttrs",
+			promoteResourceAttributes: nil,
+			ignoreAttrs:               []string{model.MetricNameLabel},
+			expectedLabels: labels.FromStrings(
+				"__name__", "test_metric",
+				"instance", "service ID",
+				"job", "service name",
+				"metric_attr", "metric value",
+				"metric_attr_other", "metric value other",
+			),
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -423,6 +436,108 @@ func TestCreateAttributes(t *testing.T) {
 			testutil.RequireEqual(t, tc.expectedLabels, lbls)
 		})
 	}
+
+	// Test that __name__ attributes in OTLP data are filtered out to prevent
+	// duplicate labels.
+	t.Run("__name__ attribute in OTLP data is filtered", func(t *testing.T) {
+		resource := pcommon.NewResource()
+		resource.Attributes().PutStr("service.name", "test-service")
+		resource.Attributes().PutStr("service.instance.id", "test-instance")
+
+		// Create attributes with __name__ to simulate problematic OTLP data.
+		attrsWithNameLabel := pcommon.NewMap()
+		attrsWithNameLabel.PutStr("__name__", "wrong_metric_name")
+		attrsWithNameLabel.PutStr("other_attr", "value")
+
+		mockAppender := &mockCombinedAppender{}
+		c := NewPrometheusConverter(mockAppender)
+		settings := Settings{}
+
+		require.NoError(t, c.setResourceContext(resource, settings))
+		require.NoError(t, c.setScopeContext(scope{}, settings))
+
+		// Call createAttributes with reservedLabelNames to filter __name__.
+		lbls, err := c.createAttributes(
+			attrsWithNameLabel,
+			settings,
+			reservedLabelNames,
+			true,
+			Metadata{},
+			model.MetricNameLabel, "correct_metric_name",
+		)
+		require.NoError(t, err)
+
+		// Verify there's exactly one __name__ label with the correct value.
+		nameCount := 0
+		var nameValue string
+		lbls.Range(func(l labels.Label) {
+			if l.Name == model.MetricNameLabel {
+				nameCount++
+				nameValue = l.Value
+			}
+		})
+
+		require.Equal(t, 1, nameCount)
+		require.Equal(t, "correct_metric_name", nameValue)
+		require.Equal(t, "value", lbls.Get("other_attr"))
+	})
+
+	// Test that __type__ and __unit__ attributes in OTLP data are overwritten
+	// by auto-generated labels from metadata when EnableTypeAndUnitLabels is true.
+	t.Run("__type__ and __unit__ attributes are overwritten by metadata", func(t *testing.T) {
+		resource := pcommon.NewResource()
+		resource.Attributes().PutStr("service.name", "test-service")
+		resource.Attributes().PutStr("service.instance.id", "test-instance")
+
+		// Create attributes with __type__ and __unit__ to simulate problematic OTLP data.
+		attrsWithTypeAndUnit := pcommon.NewMap()
+		attrsWithTypeAndUnit.PutStr(model.MetricTypeLabel, "wrong_type")
+		attrsWithTypeAndUnit.PutStr(model.MetricUnitLabel, "wrong_unit")
+		attrsWithTypeAndUnit.PutStr("other_attr", "value")
+
+		mockAppender := &mockCombinedAppender{}
+		c := NewPrometheusConverter(mockAppender)
+		settings := Settings{EnableTypeAndUnitLabels: true}
+
+		require.NoError(t, c.setResourceContext(resource, settings))
+		require.NoError(t, c.setScopeContext(scope{}, settings))
+
+		// Call createAttributes with Metadata containing correct Type and Unit.
+		lbls, err := c.createAttributes(
+			attrsWithTypeAndUnit,
+			settings,
+			reservedLabelNames,
+			true,
+			Metadata{Metadata: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "seconds"}},
+			model.MetricNameLabel, "test_metric",
+		)
+		require.NoError(t, err)
+
+		// Verify there's exactly one __type__ label with the correct value (from metadata).
+		typeCount := 0
+		var typeValue string
+		lbls.Range(func(l labels.Label) {
+			if l.Name == model.MetricTypeLabel {
+				typeCount++
+				typeValue = l.Value
+			}
+		})
+		require.Equal(t, 1, typeCount)
+		require.Equal(t, "gauge", typeValue)
+
+		// Verify there's exactly one __unit__ label with the correct value (from metadata).
+		unitCount := 0
+		var unitValue string
+		lbls.Range(func(l labels.Label) {
+			if l.Name == model.MetricUnitLabel {
+				unitCount++
+				unitValue = l.Value
+			}
+		})
+		require.Equal(t, 1, unitCount)
+		require.Equal(t, "seconds", unitValue)
+		require.Equal(t, "value", lbls.Get("other_attr"))
+	})
 }
 
 func Test_convertTimeStamp(t *testing.T) {

storage/remote/otlptranslator/prometheusremotewrite/histograms.go

@@ -53,7 +53,7 @@ func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Cont
 		lbls, err := c.createAttributes(
 			pt.Attributes(),
 			settings,
-			nil,
+			reservedLabelNames,
 			true,
 			meta,
 			model.MetricNameLabel,
@@ -269,7 +269,7 @@ func (c *PrometheusConverter) addCustomBucketsHistogramDataPoints(ctx context.Co
 		lbls, err := c.createAttributes(
 			pt.Attributes(),
 			settings,
-			nil,
+			reservedLabelNames,
 			true,
 			meta,
 			model.MetricNameLabel,

storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go

@@ -38,7 +38,7 @@ func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, data
 		labels, err := c.createAttributes(
 			pt.Attributes(),
 			settings,
-			nil,
+			reservedLabelNames,
 			true,
 			meta,
 			model.MetricNameLabel,
@@ -79,14 +79,14 @@ func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPo
 		lbls, err := c.createAttributes(
 			pt.Attributes(),
 			settings,
-			nil,
+			reservedLabelNames,
 			true,
 			meta,
 			model.MetricNameLabel,
 			meta.MetricFamilyName,
 		)
 		if err != nil {
-			return nil
+			return err
 		}
 		var val float64
 		switch pt.ValueType() {

storage/remote/queue_manager.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"log/slog"
 	"math"
+	"slices"
 	"strconv"
 	"sync"
 	"time"
@@ -2105,12 +2106,11 @@ func setAtomicToNewer(value *atomic.Int64, newValue int64) (previous int64, upda
 
 func buildTimeSeries(timeSeries []prompb.TimeSeries, filter func(prompb.TimeSeries) bool) ([]prompb.TimeSeries, *timeSeriesStats) {
 	stats := newTimeSeriesStats()
-	keepIdx := 0
 
-	for i, ts := range timeSeries {
+	timeSeries = slices.DeleteFunc(timeSeries, func(ts prompb.TimeSeries) bool {
 		if filter != nil && filter(ts) {
 			stats.recordDropped(len(ts.Samples) > 0, len(ts.Exemplars) > 0, len(ts.Histograms) > 0)
-			continue
+			return true
 		}
 
 		// At the moment we only ever append a TimeSeries with a single sample or exemplar in it.
@@ -2123,16 +2123,10 @@ func buildTimeSeries(timeSeries []prompb.TimeSeries, filter func(prompb.TimeSeri
 		if len(ts.Histograms) > 0 {
 			stats.updateTimestamp(ts.Histograms[0].Timestamp)
 		}
+		return false
+	})
 
-		if i != keepIdx {
-			// We have to swap the kept timeseries with the one which should be dropped.
-			// Copying any elements within timeSeries could cause data corruptions when reusing the slice in a next batch (shards.populateTimeSeries).
-			timeSeries[keepIdx], timeSeries[i] = timeSeries[i], timeSeries[keepIdx]
-		}
-		keepIdx++
-	}
-
-	return timeSeries[:keepIdx], stats
+	return timeSeries, stats
 }
 
 func buildWriteRequest(logger *slog.Logger, timeSeries []prompb.TimeSeries, metadata []prompb.MetricMetadata, pBuf *proto.Buffer, filter func(prompb.TimeSeries) bool, buf compression.EncodeBuffer, compr compression.Type) (_ []byte, highest, lowest int64, _ error) {

storage/remote/queue_manager_test.go

@@ -871,7 +871,7 @@ func createTimeseries(numSamples, numSeries int, extraLabels ...labels.Label) ([
 	return samples, series
 }
 
-func createProtoTimeseriesWithOld(numSamples, baseTs int64, _ ...labels.Label) []prompb.TimeSeries {
+func createProtoTimeseriesWithOld(numSamples, baseTs int64) []prompb.TimeSeries {
 	samples := make([]prompb.TimeSeries, numSamples)
 	// use a fixed rand source so tests are consistent
 	r := rand.New(rand.NewSource(99))
@@ -2365,8 +2365,14 @@ func BenchmarkBuildTimeSeries(b *testing.B) {
 	// Send one sample per series, which is the typical remote_write case
 	const numSamples = 10000
 	filter := func(ts prompb.TimeSeries) bool { return filterTsLimit(99, ts) }
+	originalSamples := createProtoTimeseriesWithOld(numSamples, 100)
+
+	b.ReportAllocs()
 	for b.Loop() {
-		samples := createProtoTimeseriesWithOld(numSamples, 100, extraLabels...)
+		b.StopTimer()
+		samples := make([]prompb.TimeSeries, len(originalSamples))
+		copy(samples, originalSamples)
+		b.StartTimer()
 		result, _ := buildTimeSeries(samples, filter)
 		require.NotNil(b, result)
 	}

storage/series.go

@@ -447,7 +447,17 @@ func (e errChunksIterator) Err() error    { return e.err }
 // ExpandSamples iterates over all samples in the iterator, buffering all in slice.
 // Optionally it takes samples constructor, useful when you want to compare sample slices with different
 // sample implementations. if nil, sample type from this package will be used.
+// For float sample, NaN values are replaced with -42.
 func ExpandSamples(iter chunkenc.Iterator, newSampleFn func(st, t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) {
+	return expandSamples(iter, true, newSampleFn)
+}
+
+// ExpandSamplesWithoutReplacingNaNs is same as ExpandSamples but it does not replace float sample NaN values with anything.
+func ExpandSamplesWithoutReplacingNaNs(iter chunkenc.Iterator, newSampleFn func(st, t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) {
+	return expandSamples(iter, false, newSampleFn)
+}
+
+func expandSamples(iter chunkenc.Iterator, replaceNaN bool, newSampleFn func(st, t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) {
 	if newSampleFn == nil {
 		newSampleFn = func(st, t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample {
 			switch {
@@ -470,7 +480,7 @@ func ExpandSamples(iter chunkenc.Iterator, newSampleFn func(st, t int64, f float
 			t, f := iter.At()
 			st := iter.AtST()
 			// NaNs can't be compared normally, so substitute for another value.
-			if math.IsNaN(f) {
+			if replaceNaN && math.IsNaN(f) {
 				f = -42
 			}
 			result = append(result, newSampleFn(st, t, f, nil, nil))

tsdb/agent/db.go

@@ -92,6 +92,11 @@ type Options struct {
 	// NOTE(bwplotka): This feature might be deprecated and removed once PROM-60
 	// is implemented.
 	EnableSTAsZeroSample bool
+
+	// EnableSTStorage determines whether agent DB should write a Start Timestamp (ST)
+	// per sample to WAL.
+	// TODO(bwplotka): Implement this option as per PROM-60, currently it's noop.
+	EnableSTStorage bool
 }
 
 // DefaultOptions used for the WAL storage. They are reasonable for setups using

tsdb/block.go

@@ -228,6 +228,18 @@ func (bm *BlockMetaCompaction) FromOutOfOrder() bool {
 	return slices.Contains(bm.Hints, CompactionHintFromOutOfOrder)
 }
 
+func (bm *BlockMetaCompaction) SetStaleSeries() {
+	if bm.FromStaleSeries() {
+		return
+	}
+	bm.Hints = append(bm.Hints, CompactionHintFromStaleSeries)
+	slices.Sort(bm.Hints)
+}
+
+func (bm *BlockMetaCompaction) FromStaleSeries() bool {
+	return slices.Contains(bm.Hints, CompactionHintFromStaleSeries)
+}
+
 const (
 	indexFilename = "index"
 	metaFilename  = "meta.json"
@@ -236,6 +248,10 @@ const (
 	// CompactionHintFromOutOfOrder is a hint noting that the block
 	// was created from out-of-order chunks.
 	CompactionHintFromOutOfOrder = "from-out-of-order"
+
+	// CompactionHintFromStaleSeries is a hint noting that the block
+	// was created from stale series.
+	CompactionHintFromStaleSeries = "from-stale-series"
 )
 
 func chunkDir(dir string) string { return filepath.Join(dir, "chunks") }

tsdb/compact.go

@@ -263,6 +263,13 @@ func (c *LeveledCompactor) Plan(dir string) ([]string, error) {
 			return nil, err
 		}
 		if c.blockExcludeFunc != nil && c.blockExcludeFunc(meta) {
+			// Compactions work from oldest to newest, uploads do the same (usually).
+			// If you continue here you'll skip compactions on this one block, but:
+			// * all further blocks are NOT yet uploaded
+			// * some or all further blocks are uploaded
+			//
+			// If we continue and there are newer blocks to pick from,
+			// then you will compact in a non-continuous way, leaving gaps of individual un-compacted blocks.
 			break
 		}
 		dms = append(dms, dirMeta{dir, meta})
@@ -598,6 +605,9 @@ func (c *LeveledCompactor) Write(dest string, b BlockReader, mint, maxt int64, b
 		if base.Compaction.FromOutOfOrder() {
 			meta.Compaction.SetOutOfOrder()
 		}
+		if base.Compaction.FromStaleSeries() {
+			meta.Compaction.SetStaleSeries()
+		}
 	}
 
 	err := c.write(dest, meta, DefaultBlockPopulator{}, b)

tsdb/compact_test.go

@@ -173,214 +173,274 @@ func TestNoPanicFor0Tombstones(t *testing.T) {
 	c.plan(metas)
 }
 
-func TestLeveledCompactor_plan(t *testing.T) {
-	// This mimics our default ExponentialBlockRanges with min block size equals to 20.
-	compactor, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{
-		20,
-		60,
-		180,
-		540,
-		1620,
-	}, nil, nil)
-	require.NoError(t, err)
+func TestLeveledCompactor(t *testing.T) {
+	// Tests for the private plan() method.
+	t.Run("plan", func(t *testing.T) {
+		// This mimics our default ExponentialBlockRanges with min block size equals to 20.
+		compactor, err := NewLeveledCompactor(context.Background(), nil, nil, []int64{
+			20,
+			60,
+			180,
+			540,
+			1620,
+		}, nil, nil)
+		require.NoError(t, err)
 
-	cases := map[string]struct {
-		metas    []dirMeta
-		expected []string
-	}{
-		"Outside Range": {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
+		cases := map[string]struct {
+			metas    []dirMeta
+			expected []string
+		}{
+			"Outside Range": {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+				},
+				expected: nil,
 			},
-			expected: nil,
-		},
-		"We should wait for four blocks of size 20 to appear before compacting.": {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
-				metaRange("2", 20, 40, nil),
+			"We should wait for four blocks of size 20 to appear before compacting.": {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+					metaRange("2", 20, 40, nil),
+				},
+				expected: nil,
 			},
-			expected: nil,
-		},
-		`We should wait for a next block of size 20 to appear before compacting
-		the existing ones. We have three, but we ignore the fresh one from WAl`: {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
-				metaRange("2", 20, 40, nil),
-				metaRange("3", 40, 60, nil),
+			`We should wait for a next block of size 20 to appear before compacting
+			the existing ones. We have three, but we ignore the fresh one from WAl`: {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+					metaRange("2", 20, 40, nil),
+					metaRange("3", 40, 60, nil),
+				},
+				expected: nil,
 			},
-			expected: nil,
-		},
-		"Block to fill the entire parent range appeared – should be compacted": {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
-				metaRange("2", 20, 40, nil),
-				metaRange("3", 40, 60, nil),
-				metaRange("4", 60, 80, nil),
+			"Block to fill the entire parent range appeared – should be compacted": {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+					metaRange("2", 20, 40, nil),
+					metaRange("3", 40, 60, nil),
+					metaRange("4", 60, 80, nil),
+				},
+				expected: []string{"1", "2", "3"},
 			},
-			expected: []string{"1", "2", "3"},
-		},
-		`Block for the next parent range appeared with gap with size 20. Nothing will happen in the first one
-		anymore but we ignore fresh one still, so no compaction`: {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
-				metaRange("2", 20, 40, nil),
-				metaRange("3", 60, 80, nil),
+			`Block for the next parent range appeared with gap with size 20. Nothing will happen in the first one
+			anymore but we ignore fresh one still, so no compaction`: {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+					metaRange("2", 20, 40, nil),
+					metaRange("3", 60, 80, nil),
+				},
+				expected: nil,
 			},
-			expected: nil,
-		},
-		`Block for the next parent range appeared, and we have a gap with size 20 between second and third block.
-		We will not get this missed gap anymore and we should compact just these two.`: {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
-				metaRange("2", 20, 40, nil),
-				metaRange("3", 60, 80, nil),
-				metaRange("4", 80, 100, nil),
+			`Block for the next parent range appeared, and we have a gap with size 20 between second and third block.
+			We will not get this missed gap anymore and we should compact just these two.`: {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+					metaRange("2", 20, 40, nil),
+					metaRange("3", 60, 80, nil),
+					metaRange("4", 80, 100, nil),
+				},
+				expected: []string{"1", "2"},
 			},
-			expected: []string{"1", "2"},
-		},
-		"We have 20, 20, 20, 60, 60 range blocks. '5' is marked as fresh one": {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
-				metaRange("2", 20, 40, nil),
-				metaRange("3", 40, 60, nil),
-				metaRange("4", 60, 120, nil),
-				metaRange("5", 120, 180, nil),
+			"We have 20, 20, 20, 60, 60 range blocks. '5' is marked as fresh one": {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+					metaRange("2", 20, 40, nil),
+					metaRange("3", 40, 60, nil),
+					metaRange("4", 60, 120, nil),
+					metaRange("5", 120, 180, nil),
+				},
+				expected: []string{"1", "2", "3"},
 			},
-			expected: []string{"1", "2", "3"},
-		},
-		"We have 20, 60, 20, 60, 240 range blocks. We can compact 20 + 60 + 60": {
-			metas: []dirMeta{
-				metaRange("2", 20, 40, nil),
-				metaRange("4", 60, 120, nil),
-				metaRange("5", 960, 980, nil), // Fresh one.
-				metaRange("6", 120, 180, nil),
-				metaRange("7", 720, 960, nil),
+			"We have 20, 60, 20, 60, 240 range blocks. We can compact 20 + 60 + 60": {
+				metas: []dirMeta{
+					metaRange("2", 20, 40, nil),
+					metaRange("4", 60, 120, nil),
+					metaRange("5", 960, 980, nil), // Fresh one.
+					metaRange("6", 120, 180, nil),
+					metaRange("7", 720, 960, nil),
+				},
+				expected: []string{"2", "4", "6"},
 			},
-			expected: []string{"2", "4", "6"},
-		},
-		"Do not select large blocks that have many tombstones when there is no fresh block": {
-			metas: []dirMeta{
-				metaRange("1", 0, 540, &BlockStats{
-					NumSeries:     10,
-					NumTombstones: 3,
-				}),
+			"Do not select large blocks that have many tombstones when there is no fresh block": {
+				metas: []dirMeta{
+					metaRange("1", 0, 540, &BlockStats{
+						NumSeries:     10,
+						NumTombstones: 3,
+					}),
+				},
+				expected: nil,
 			},
-			expected: nil,
-		},
-		"Select large blocks that have many tombstones when fresh appears": {
-			metas: []dirMeta{
-				metaRange("1", 0, 540, &BlockStats{
-					NumSeries:     10,
-					NumTombstones: 3,
-				}),
-				metaRange("2", 540, 560, nil),
+			"Select large blocks that have many tombstones when fresh appears": {
+				metas: []dirMeta{
+					metaRange("1", 0, 540, &BlockStats{
+						NumSeries:     10,
+						NumTombstones: 3,
+					}),
+					metaRange("2", 540, 560, nil),
+				},
+				expected: []string{"1"},
 			},
-			expected: []string{"1"},
-		},
-		"For small blocks, do not compact tombstones, even when fresh appears.": {
-			metas: []dirMeta{
-				metaRange("1", 0, 60, &BlockStats{
-					NumSeries:     10,
-					NumTombstones: 3,
-				}),
-				metaRange("2", 60, 80, nil),
+			"For small blocks, do not compact tombstones, even when fresh appears.": {
+				metas: []dirMeta{
+					metaRange("1", 0, 60, &BlockStats{
+						NumSeries:     10,
+						NumTombstones: 3,
+					}),
+					metaRange("2", 60, 80, nil),
+				},
+				expected: nil,
 			},
-			expected: nil,
-		},
-		`Regression test: we were stuck in a compact loop where we always recompacted
-		the same block when tombstones and series counts were zero`: {
-			metas: []dirMeta{
-				metaRange("1", 0, 540, &BlockStats{
-					NumSeries:     0,
-					NumTombstones: 0,
-				}),
-				metaRange("2", 540, 560, nil),
+			`Regression test: we were stuck in a compact loop where we always recompacted
+			the same block when tombstones and series counts were zero`: {
+				metas: []dirMeta{
+					metaRange("1", 0, 540, &BlockStats{
+						NumSeries:     0,
+						NumTombstones: 0,
+					}),
+					metaRange("2", 540, 560, nil),
+				},
+				expected: nil,
 			},
-			expected: nil,
-		},
-		`Regression test: we were wrongly assuming that new block is fresh from WAL when its ULID is newest.
-		We need to actually look on max time instead.
+			`Regression test: we were wrongly assuming that new block is fresh from WAL when its ULID is newest.
+			We need to actually look on max time instead.
 
-		With previous, wrong approach "8" block was ignored, so we were wrongly compacting 5 and 7 and introducing
-		block overlaps`: {
-			metas: []dirMeta{
-				metaRange("5", 0, 360, nil),
-				metaRange("6", 540, 560, nil), // Fresh one.
-				metaRange("7", 360, 420, nil),
-				metaRange("8", 420, 540, nil),
+			With previous, wrong approach "8" block was ignored, so we were wrongly compacting 5 and 7 and introducing
+			block overlaps`: {
+				metas: []dirMeta{
+					metaRange("5", 0, 360, nil),
+					metaRange("6", 540, 560, nil), // Fresh one.
+					metaRange("7", 360, 420, nil),
+					metaRange("8", 420, 540, nil),
+				},
+				expected: []string{"7", "8"},
 			},
-			expected: []string{"7", "8"},
-		},
-		// |--------------|
-		//               |----------------|
-		//                                |--------------|
-		"Overlapping blocks 1": {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
-				metaRange("2", 19, 40, nil),
-				metaRange("3", 40, 60, nil),
+			// |--------------|
+			//               |----------------|
+			//                                |--------------|
+			"Overlapping blocks 1": {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+					metaRange("2", 19, 40, nil),
+					metaRange("3", 40, 60, nil),
+				},
+				expected: []string{"1", "2"},
 			},
-			expected: []string{"1", "2"},
-		},
-		// |--------------|
-		//                |--------------|
-		//                        |--------------|
-		"Overlapping blocks 2": {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
-				metaRange("2", 20, 40, nil),
-				metaRange("3", 30, 50, nil),
+			// |--------------|
+			//                |--------------|
+			//                        |--------------|
+			"Overlapping blocks 2": {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+					metaRange("2", 20, 40, nil),
+					metaRange("3", 30, 50, nil),
+				},
+				expected: []string{"2", "3"},
 			},
-			expected: []string{"2", "3"},
-		},
-		// |--------------|
-		//         |---------------------|
-		//                       |--------------|
-		"Overlapping blocks 3": {
-			metas: []dirMeta{
-				metaRange("1", 0, 20, nil),
-				metaRange("2", 10, 40, nil),
-				metaRange("3", 30, 50, nil),
+			// |--------------|
+			//         |---------------------|
+			//                       |--------------|
+			"Overlapping blocks 3": {
+				metas: []dirMeta{
+					metaRange("1", 0, 20, nil),
+					metaRange("2", 10, 40, nil),
+					metaRange("3", 30, 50, nil),
+				},
+				expected: []string{"1", "2", "3"},
 			},
-			expected: []string{"1", "2", "3"},
-		},
-		// |--------------|
-		//               |--------------------------------|
-		//                |--------------|
-		//                               |--------------|
-		"Overlapping blocks 4": {
-			metas: []dirMeta{
-				metaRange("5", 0, 360, nil),
-				metaRange("6", 340, 560, nil),
-				metaRange("7", 360, 420, nil),
-				metaRange("8", 420, 540, nil),
+			// |--------------|
+			//               |--------------------------------|
+			//                |--------------|
+			//                               |--------------|
+			"Overlapping blocks 4": {
+				metas: []dirMeta{
+					metaRange("5", 0, 360, nil),
+					metaRange("6", 340, 560, nil),
+					metaRange("7", 360, 420, nil),
+					metaRange("8", 420, 540, nil),
+				},
+				expected: []string{"5", "6", "7", "8"},
 			},
-			expected: []string{"5", "6", "7", "8"},
-		},
-		// |--------------|
-		//               |--------------|
-		//                                            |--------------|
-		//                                                          |--------------|
-		"Overlapping blocks 5": {
-			metas: []dirMeta{
-				metaRange("1", 0, 10, nil),
-				metaRange("2", 9, 20, nil),
-				metaRange("3", 30, 40, nil),
-				metaRange("4", 39, 50, nil),
+			// |--------------|
+			//               |--------------|
+			//                                            |--------------|
+			//                                                          |--------------|
+			"Overlapping blocks 5": {
+				metas: []dirMeta{
+					metaRange("1", 0, 10, nil),
+					metaRange("2", 9, 20, nil),
+					metaRange("3", 30, 40, nil),
+					metaRange("4", 39, 50, nil),
+				},
+				expected: []string{"1", "2"},
 			},
-			expected: []string{"1", "2"},
-		},
-	}
-
-	for title, c := range cases {
-		if !t.Run(title, func(t *testing.T) {
-			res, err := compactor.plan(c.metas)
-			require.NoError(t, err)
-			require.Equal(t, c.expected, res)
-		}) {
-			return
 		}
-	}
+
+		for title, c := range cases {
+			if !t.Run(title, func(t *testing.T) {
+				res, err := compactor.plan(c.metas)
+				require.NoError(t, err)
+				require.Equal(t, c.expected, res)
+			}) {
+				return
+			}
+		}
+	})
+
+	// Tests for the public Plan() method.
+	t.Run("Plan", func(t *testing.T) {
+		// Verify that when a BlockExcludeFilter excludes a block in the middle of
+		// the list, subsequent blocks are not processed.
+		t.Run("BlockExcludeFilter stops iteration", func(t *testing.T) {
+			dir := t.TempDir()
+
+			// Create 4 blocks with sequential ULIDs.
+			block1ULID := ulid.MustNew(1, nil)
+			block2ULID := ulid.MustNew(2, nil)
+			block3ULID := ulid.MustNew(3, nil)
+			block4ULID := ulid.MustNew(4, nil)
+
+			for i, uid := range []ulid.ULID{block1ULID, block2ULID, block3ULID, block4ULID} {
+				blockDir := filepath.Join(dir, uid.String())
+				require.NoError(t, os.MkdirAll(blockDir, 0o777))
+
+				meta := &BlockMeta{
+					ULID:    uid,
+					MinTime: int64(i * 10),
+					MaxTime: int64((i + 1) * 10),
+				}
+				meta.Compaction.Level = 1
+				_, err := writeMetaFile(promslog.NewNopLogger(), blockDir, meta)
+				require.NoError(t, err)
+			}
+
+			// Track which blocks were evaluated by the exclude function.
+			var evaluatedBlocks []ulid.ULID
+			excludeFunc := func(meta *BlockMeta) bool {
+				evaluatedBlocks = append(evaluatedBlocks, meta.ULID)
+				return meta.ULID == block2ULID
+			}
+
+			c, err := NewLeveledCompactorWithOptions(
+				context.Background(),
+				nil,
+				promslog.NewNopLogger(),
+				[]int64{20},
+				chunkenc.NewPool(),
+				LeveledCompactorOptions{
+					BlockExcludeFilter:          excludeFunc,
+					EnableOverlappingCompaction: true,
+				},
+			)
+			require.NoError(t, err)
+
+			// Plan should evaluate all blocks.
+			_, err = c.Plan(dir)
+			require.NoError(t, err)
+
+			require.Len(t, evaluatedBlocks, 2, "Expected only 2 blocks to be evaluated")
+			require.Contains(t, evaluatedBlocks, block1ULID)
+			require.Contains(t, evaluatedBlocks, block2ULID)
+		})
+	})
 }
 
 func TestRangeWithFailedCompactionWontGetSelected(t *testing.T) {

tsdb/db.go

@@ -100,6 +100,10 @@ func DefaultOptions() *Options {
 
 // Options of the DB storage.
 type Options struct {
+	// staleSeriesCompactionThreshold is same as below option with same name, but is atomic so that we can do live updates without locks.
+	// This is the one that must be used by the code.
+	staleSeriesCompactionThreshold atomic.Float64
+
 	// Segments (wal files) max size.
 	// WALSegmentSize = 0, segment size is default size.
 	// WALSegmentSize > 0, segment size is WALSegmentSize.
@@ -231,6 +235,11 @@ type Options struct {
 	// is implemented.
 	EnableSTAsZeroSample bool
 
+	// EnableSTStorage determines whether TSDB should write a Start Timestamp (ST)
+	// per sample to WAL.
+	// TODO(bwplotka): Implement this option as per PROM-60, currently it's noop.
+	EnableSTStorage bool
+
 	// EnableMetadataWALRecords represents 'metadata-wal-records' feature flag.
 	// NOTE(bwplotka): This feature might be deprecated and removed once PROM-60
 	// is implemented.
@@ -245,6 +254,10 @@ type Options struct {
 
 	// FeatureRegistry is used to register TSDB features.
 	FeatureRegistry features.Collector
+
+	// StaleSeriesCompactionThreshold is a number between 0.0-1.0 indicating the % of stale series in
+	// the in-memory Head block. If the % of stale series crosses this threshold, stale series compaction is run immediately.
+	StaleSeriesCompactionThreshold float64
 }
 
 type NewCompactorFunc func(ctx context.Context, r prometheus.Registerer, l *slog.Logger, ranges []int64, pool chunkenc.Pool, opts *Options) (Compactor, error)
@@ -305,6 +318,10 @@ type DB struct {
 	// out-of-order compaction and vertical queries.
 	oooWasEnabled atomic.Bool
 
+	// lastHeadCompactionTime is the last wall clock time when the head block compaction was started,
+	// irrespective of success or failure. This does not include out-of-order compaction and stale series compaction.
+	lastHeadCompactionTime time.Time
+
 	writeNotified wlog.WriteNotified
 
 	registerer prometheus.Registerer
@@ -857,6 +874,8 @@ func validateOpts(opts *Options, rngs []int64) (*Options, []int64) {
 		// configured maximum block duration.
 		rngs = ExponentialBlockRanges(opts.MinBlockDuration, 10, 3)
 	}
+
+	opts.staleSeriesCompactionThreshold.Store(opts.StaleSeriesCompactionThreshold)
 	return opts, rngs
 }
 
@@ -1151,6 +1170,29 @@ func (db *DB) run(ctx context.Context) {
 			}
 			// We attempt mmapping of head chunks regularly.
 			db.head.mmapHeadChunks()
+
+			numStaleSeries, numSeries := db.Head().NumStaleSeries(), db.Head().NumSeries()
+			if db.autoCompact && numSeries > 0 && db.opts.staleSeriesCompactionThreshold.Load() > 0 {
+				staleSeriesRatio := float64(numStaleSeries) / float64(numSeries)
+				if staleSeriesRatio >= db.opts.staleSeriesCompactionThreshold.Load() {
+					nextCompactionIsSoon := false
+					if !db.lastHeadCompactionTime.IsZero() {
+						compactionInterval := time.Duration(db.head.chunkRange.Load()) * time.Millisecond
+						nextEstimatedCompactionTime := db.lastHeadCompactionTime.Add(compactionInterval)
+						if time.Now().Add(10 * time.Minute).After(nextEstimatedCompactionTime) {
+							// Next compaction is starting within next 10 mins.
+							nextCompactionIsSoon = true
+						}
+					}
+
+					if !nextCompactionIsSoon {
+						if err := db.CompactStaleHead(); err != nil {
+							db.logger.Error("immediate stale series compaction failed", "err", err)
+						}
+					}
+				}
+			}
+
 		case <-db.compactc:
 			db.metrics.compactionsTriggered.Inc()
 
@@ -1203,7 +1245,7 @@ func (db *DB) ApplyConfig(conf *config.Config) error {
 	oooTimeWindow := int64(0)
 	if conf.StorageConfig.TSDBConfig != nil {
 		oooTimeWindow = conf.StorageConfig.TSDBConfig.OutOfOrderTimeWindow
-
+		db.opts.staleSeriesCompactionThreshold.Store(conf.StorageConfig.TSDBConfig.StaleSeriesCompactionThreshold)
 		// Update retention configuration if provided.
 		if conf.StorageConfig.TSDBConfig.Retention != nil {
 			db.retentionMtx.Lock()
@@ -1217,6 +1259,8 @@ func (db *DB) ApplyConfig(conf *config.Config) error {
 			}
 			db.retentionMtx.Unlock()
 		}
+	} else {
+		db.opts.staleSeriesCompactionThreshold.Store(0)
 	}
 	if oooTimeWindow < 0 {
 		oooTimeWindow = 0
@@ -1560,6 +1604,8 @@ func (db *DB) compactOOO(dest string, oooHead *OOOCompactionHead) (_ []ulid.ULID
 // compactHead compacts the given RangeHead.
 // The db.cmtx should be held before calling this method.
 func (db *DB) compactHead(head *RangeHead) error {
+	db.lastHeadCompactionTime = time.Now()
+
 	uids, err := db.compactor.Write(db.dir, head, head.MinTime(), head.BlockMaxTime(), nil)
 	if err != nil {
 		return fmt.Errorf("persist head block: %w", err)
@@ -1583,6 +1629,52 @@ func (db *DB) compactHead(head *RangeHead) error {
 	return nil
 }
 
+func (db *DB) CompactStaleHead() error {
+	db.cmtx.Lock()
+	defer db.cmtx.Unlock()
+
+	db.logger.Info("Starting stale series compaction")
+	start := time.Now()
+
+	// We get the stale series reference first because this list can change during the compaction below.
+	// It is more efficient and easier to provide an index interface for the stale series when we have a static list.
+	staleSeriesRefs, err := db.head.SortedStaleSeriesRefsNoOOOData(context.Background())
+	if err != nil {
+		return err
+	}
+	meta := &BlockMeta{}
+	meta.Compaction.SetStaleSeries()
+	mint, maxt := db.head.opts.ChunkRange*(db.head.MinTime()/db.head.opts.ChunkRange), db.head.MaxTime()
+	for ; mint < maxt; mint += db.head.chunkRange.Load() {
+		staleHead := NewStaleHead(db.Head(), mint, mint+db.head.chunkRange.Load()-1, staleSeriesRefs)
+
+		uids, err := db.compactor.Write(db.dir, staleHead, staleHead.MinTime(), staleHead.BlockMaxTime(), meta)
+		if err != nil {
+			return fmt.Errorf("persist stale head: %w", err)
+		}
+
+		db.logger.Info("Stale series block created", "ulids", fmt.Sprintf("%v", uids), "min_time", mint, "max_time", maxt)
+
+		if err := db.reloadBlocks(); err != nil {
+			errs := []error{fmt.Errorf("reloadBlocks blocks: %w", err)}
+			for _, uid := range uids {
+				if errRemoveAll := os.RemoveAll(filepath.Join(db.dir, uid.String())); errRemoveAll != nil {
+					errs = append(errs, fmt.Errorf("delete persisted stale head block after failed db reloadBlocks:%s: %w", uid, errRemoveAll))
+				}
+			}
+			return errors.Join(errs...)
+		}
+	}
+
+	if err := db.head.truncateStaleSeries(staleSeriesRefs, maxt); err != nil {
+		return fmt.Errorf("head truncate: %w", err)
+	}
+	db.head.RebuildSymbolTable(db.logger)
+
+	db.logger.Info("Ending stale series compaction", "num_series", meta.Stats.NumSeries, "duration", time.Since(start))
+	return nil
+}
+
 // compactBlocks compacts all the eligible on-disk blocks.
 // The db.cmtx should be held before calling this method.
 func (db *DB) compactBlocks() (err error) {
@@ -2042,7 +2134,7 @@ func (db *DB) inOrderBlocksMaxTime() (maxt int64, ok bool) {
 	maxt, ok = int64(math.MinInt64), false
 	// If blocks are overlapping, last block might not have the max time. So check all blocks.
 	for _, b := range db.Blocks() {
-		if !b.meta.Compaction.FromOutOfOrder() && b.meta.MaxTime > maxt {
+		if !b.meta.Compaction.FromOutOfOrder() && !b.meta.Compaction.FromStaleSeries() && b.meta.MaxTime > maxt {
 			ok = true
 			maxt = b.meta.MaxTime
 		}

tsdb/db_test.go

@@ -52,6 +52,7 @@ import (
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/metadata"
+	"github.com/prometheus/prometheus/model/value"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/storage/remote"
@@ -145,6 +146,16 @@ func TestDBClose_AfterClose(t *testing.T) {
 
 // query runs a matcher query against the querier and fully expands its data.
 func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[string][]chunks.Sample {
+	return queryHelper(t, q, true, matchers...)
+}
+
+// queryWithoutReplacingNaNs runs a matcher query against the querier and fully expands its data.
+func queryWithoutReplacingNaNs(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[string][]chunks.Sample {
+	return queryHelper(t, q, false, matchers...)
+}
+
+// queryHelper runs a matcher query against the querier and fully expands its data.
+func queryHelper(t testing.TB, q storage.Querier, withNaNReplacement bool, matchers ...*labels.Matcher) map[string][]chunks.Sample {
 	ss := q.Select(context.Background(), false, nil, matchers...)
 	defer func() {
 		require.NoError(t, q.Close())
@@ -156,7 +167,13 @@ func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[str
 		series := ss.At()
 
 		it = series.Iterator(it)
-		samples, err := storage.ExpandSamples(it, newSample)
+		var samples []chunks.Sample
+		var err error
+		if withNaNReplacement {
+			samples, err = storage.ExpandSamples(it, newSample)
+		} else {
+			samples, err = storage.ExpandSamplesWithoutReplacingNaNs(it, newSample)
+		}
 		require.NoError(t, err)
 		require.NoError(t, it.Err())
 
@@ -2610,7 +2627,7 @@ func TestDBReadOnly_FlushWAL(t *testing.T) {
 		db.DisableCompactions()
 		app := db.Appender(ctx)
 		maxt = 1000
-		for i := 0; i < maxt; i++ {
+		for i := range maxt {
 			_, err := app.Append(0, labels.FromStrings(defaultLabelName, "flush"), int64(i), 1.0)
 			require.NoError(t, err)
 		}
@@ -9323,3 +9340,248 @@ func TestBlockReloadInterval(t *testing.T) {
 		})
 	}
 }
+
+func TestStaleSeriesCompaction(t *testing.T) {
+	opts := DefaultOptions()
+	opts.MinBlockDuration = 1000
+	opts.MaxBlockDuration = 1000
+	db := newTestDB(t, withOpts(opts))
+	db.DisableCompactions()
+	t.Cleanup(func() {
+		require.NoError(t, db.Close())
+	})
+
+	var (
+		nonStaleSeries, staleSeries,
+		nonStaleHist, staleHist,
+		nonStaleFHist, staleFHist,
+		staleSeriesCrossingBoundary, staleHistCrossingBoundary, staleFHistCrossingBoundary []labels.Labels
+		numSeriesPerCategory = 1
+	)
+	for i := range numSeriesPerCategory {
+		nonStaleSeries = append(nonStaleSeries, labels.FromStrings("name", fmt.Sprintf("series%d", 1000+i)))
+		nonStaleHist = append(nonStaleHist, labels.FromStrings("name", fmt.Sprintf("series%d", 2000+i)))
+		nonStaleFHist = append(nonStaleFHist, labels.FromStrings("name", fmt.Sprintf("series%d", 3000+i)))
+
+		staleSeries = append(staleSeries, labels.FromStrings("name", fmt.Sprintf("series%d", 4000+i)))
+		staleHist = append(staleHist, labels.FromStrings("name", fmt.Sprintf("series%d", 5000+i)))
+		staleFHist = append(staleFHist, labels.FromStrings("name", fmt.Sprintf("series%d", 6000+i)))
+
+		staleSeriesCrossingBoundary = append(staleSeriesCrossingBoundary, labels.FromStrings("name", fmt.Sprintf("series%d", 7000+i)))
+		staleHistCrossingBoundary = append(staleHistCrossingBoundary, labels.FromStrings("name", fmt.Sprintf("series%d", 8000+i)))
+		staleFHistCrossingBoundary = append(staleFHistCrossingBoundary, labels.FromStrings("name", fmt.Sprintf("series%d", 9000+i)))
+	}
+
+	var (
+		v       = 10.0
+		staleV  = math.Float64frombits(value.StaleNaN)
+		h       = tsdbutil.GenerateTestHistograms(1)[0]
+		fh      = tsdbutil.GenerateTestFloatHistograms(1)[0]
+		staleH  = &histogram.Histogram{Sum: staleV}
+		staleFH = &histogram.FloatHistogram{Sum: staleV}
+	)
+
+	addNormalSamples := func(ts int64, floatSeries, histSeries, floatHistSeries []labels.Labels) {
+		app := db.Appender(context.Background())
+		for i := range len(floatSeries) {
+			_, err := app.Append(0, floatSeries[i], ts, v)
+			require.NoError(t, err)
+			_, err = app.AppendHistogram(0, histSeries[i], ts, h, nil)
+			require.NoError(t, err)
+			_, err = app.AppendHistogram(0, floatHistSeries[i], ts, nil, fh)
+			require.NoError(t, err)
+		}
+		require.NoError(t, app.Commit())
+	}
+	addStaleSamples := func(ts int64, floatSeries, histSeries, floatHistSeries []labels.Labels) {
+		app := db.Appender(context.Background())
+		for i := range len(floatSeries) {
+			_, err := app.Append(0, floatSeries[i], ts, staleV)
+			require.NoError(t, err)
+			_, err = app.AppendHistogram(0, histSeries[i], ts, staleH, nil)
+			require.NoError(t, err)
+			_, err = app.AppendHistogram(0, floatHistSeries[i], ts, nil, staleFH)
+			require.NoError(t, err)
+		}
+		require.NoError(t, app.Commit())
+	}
+
+	// Normal sample for all.
+	addNormalSamples(100, nonStaleSeries, nonStaleHist, nonStaleFHist)
+	addNormalSamples(100, staleSeries, staleHist, staleFHist)
+
+	// Stale sample for the stale series. Normal sample for the non-stale series.
+	addNormalSamples(200, nonStaleSeries, nonStaleHist, nonStaleFHist)
+	addStaleSamples(200, staleSeries, staleHist, staleFHist)
+
+	// Normal samples for the non-stale series later
+	addNormalSamples(300, nonStaleSeries, nonStaleHist, nonStaleFHist)
+
+	require.Equal(t, uint64(6*numSeriesPerCategory), db.Head().NumSeries())
+	require.Equal(t, uint64(3*numSeriesPerCategory), db.Head().NumStaleSeries())
+
+	// Series crossing block boundary and gets stale.
+	addNormalSamples(300, staleSeriesCrossingBoundary, staleHistCrossingBoundary, staleFHistCrossingBoundary)
+	addNormalSamples(700, staleSeriesCrossingBoundary, staleHistCrossingBoundary, staleFHistCrossingBoundary)
+	addNormalSamples(1100, staleSeriesCrossingBoundary, staleHistCrossingBoundary, staleFHistCrossingBoundary)
+	addStaleSamples(1200, staleSeriesCrossingBoundary, staleHistCrossingBoundary, staleFHistCrossingBoundary)
+
+	require.NoError(t, db.CompactStaleHead())
+
+	require.Equal(t, uint64(3*numSeriesPerCategory), db.Head().NumSeries())
+	require.Equal(t, uint64(0), db.Head().NumStaleSeries())
+
+	require.Len(t, db.Blocks(), 2)
+	m := db.Blocks()[0].Meta()
+	require.Equal(t, int64(0), m.MinTime)
+	require.Equal(t, int64(1000), m.MaxTime)
+	require.Truef(t, m.Compaction.FromStaleSeries(), "stale series info not found in block meta")
+	m = db.Blocks()[1].Meta()
+	require.Equal(t, int64(1000), m.MinTime)
+	require.Equal(t, int64(2000), m.MaxTime)
+	require.Truef(t, m.Compaction.FromStaleSeries(), "stale series info not found in block meta")
+
+	// To make sure that Head is not truncated based on stale series block.
+	require.NoError(t, db.reload())
+
+	nonFirstH := h.Copy()
+	nonFirstH.CounterResetHint = histogram.NotCounterReset
+	nonFirstFH := fh.Copy()
+	nonFirstFH.CounterResetHint = histogram.NotCounterReset
+
+	// Verify head block.
+	verifyHeadBlock := func() {
+		require.Equal(t, uint64(3), db.head.NumSeries())
+		require.Equal(t, uint64(0), db.head.NumStaleSeries())
+
+		expHeadQuery := make(map[string][]chunks.Sample)
+		for i := range numSeriesPerCategory {
+			expHeadQuery[fmt.Sprintf(`{name="%s"}`, nonStaleSeries[i].Get("name"))] = []chunks.Sample{
+				sample{t: 100, f: v}, sample{t: 200, f: v}, sample{t: 300, f: v},
+			}
+			expHeadQuery[fmt.Sprintf(`{name="%s"}`, nonStaleHist[i].Get("name"))] = []chunks.Sample{
+				sample{t: 100, h: h}, sample{t: 200, h: nonFirstH}, sample{t: 300, h: nonFirstH},
+			}
+			expHeadQuery[fmt.Sprintf(`{name="%s"}`, nonStaleFHist[i].Get("name"))] = []chunks.Sample{
+				sample{t: 100, fh: fh}, sample{t: 200, fh: nonFirstFH}, sample{t: 300, fh: nonFirstFH},
+			}
+		}
+
+		querier, err := NewBlockQuerier(NewRangeHead(db.head, 0, 300), 0, 300)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			querier.Close()
+		})
+		seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchRegexp, "name", "series.*"))
+		require.Equal(t, expHeadQuery, seriesSet)
+	}
+
+	verifyHeadBlock()
+
+	// Verify blocks from stale series.
+	{
+		expBlockQuery := make(map[string][]chunks.Sample)
+		for i := range numSeriesPerCategory {
+			expBlockQuery[fmt.Sprintf(`{name="%s"}`, staleSeries[i].Get("name"))] = []chunks.Sample{
+				sample{t: 100, f: v}, sample{t: 200, f: staleV},
+			}
+			expBlockQuery[fmt.Sprintf(`{name="%s"}`, staleHist[i].Get("name"))] = []chunks.Sample{
+				sample{t: 100, h: h}, sample{t: 200, h: staleH},
+			}
+			expBlockQuery[fmt.Sprintf(`{name="%s"}`, staleFHist[i].Get("name"))] = []chunks.Sample{
+				sample{t: 100, fh: fh}, sample{t: 200, fh: staleFH},
+			}
+			expBlockQuery[fmt.Sprintf(`{name="%s"}`, staleSeriesCrossingBoundary[i].Get("name"))] = []chunks.Sample{
+				sample{t: 300, f: v}, sample{t: 700, f: v}, sample{t: 1100, f: v}, sample{t: 1200, f: staleV},
+			}
+			expBlockQuery[fmt.Sprintf(`{name="%s"}`, staleHistCrossingBoundary[i].Get("name"))] = []chunks.Sample{
+				sample{t: 300, h: h}, sample{t: 700, h: nonFirstH}, sample{t: 1100, h: h}, sample{t: 1200, h: staleH},
+			}
+			expBlockQuery[fmt.Sprintf(`{name="%s"}`, staleFHistCrossingBoundary[i].Get("name"))] = []chunks.Sample{
+				sample{t: 300, fh: fh}, sample{t: 700, fh: nonFirstFH}, sample{t: 1100, fh: fh}, sample{t: 1200, fh: staleFH},
+			}
+		}
+
+		querier, err := NewBlockQuerier(db.Blocks()[0], 0, 1000)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			querier.Close()
+		})
+		seriesSet := queryWithoutReplacingNaNs(t, querier, labels.MustNewMatcher(labels.MatchRegexp, "name", "series.*"))
+
+		querier, err = NewBlockQuerier(db.Blocks()[1], 1000, 2000)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			querier.Close()
+		})
+		seriesSet2 := queryWithoutReplacingNaNs(t, querier, labels.MustNewMatcher(labels.MatchRegexp, "name", "series.*"))
+		for k, v := range seriesSet2 {
+			seriesSet[k] = append(seriesSet[k], v...)
+		}
+
+		require.Len(t, seriesSet, len(expBlockQuery))
+
+		// Compare all the samples except the stale value that needs special handling.
+		for _, category := range [][]labels.Labels{
+			staleSeries, staleHist, staleFHist,
+			staleSeriesCrossingBoundary, staleHistCrossingBoundary, staleFHistCrossingBoundary,
+		} {
+			for i := range numSeriesPerCategory {
+				seriesKey := fmt.Sprintf(`{name="%s"}`, category[i].Get("name"))
+				samples := expBlockQuery[seriesKey]
+				actSamples, exists := seriesSet[seriesKey]
+				require.Truef(t, exists, "series not found in result %s", seriesKey)
+				require.Len(t, actSamples, len(samples))
+
+				for i := range len(samples) - 1 {
+					require.Equal(t, samples[i], actSamples[i])
+				}
+
+				l := len(samples) - 1
+				require.Equal(t, samples[l].T(), actSamples[l].T())
+				switch {
+				case value.IsStaleNaN(samples[l].F()):
+					require.True(t, value.IsStaleNaN(actSamples[l].F()))
+				case samples[l].H() != nil:
+					require.True(t, value.IsStaleNaN(actSamples[l].H().Sum))
+				default:
+					require.True(t, value.IsStaleNaN(actSamples[l].FH().Sum))
+				}
+			}
+		}
+	}
+
+	{
+		// Restart DB and verify that stale series were discarded from WAL replay.
+		require.NoError(t, db.Close())
+		var err error
+		db, err = Open(db.Dir(), db.logger, db.registerer, db.opts, nil)
+		require.NoError(t, err)
+
+		verifyHeadBlock()
+	}
+}
+
+// TestStaleSeriesCompactionWithZeroSeries verifies that CompactStaleHead handles
+// an empty head (0 series) gracefully without division by zero or incorrectly
+// triggering compaction. This is a regression test for issue #17949.
+func TestStaleSeriesCompactionWithZeroSeries(t *testing.T) {
+	opts := DefaultOptions()
+	opts.MinBlockDuration = 1000
+	opts.MaxBlockDuration = 1000
+	db := newTestDB(t, withOpts(opts))
+	db.DisableCompactions()
+	t.Cleanup(func() {
+		require.NoError(t, db.Close())
+	})
+
+	// Verify the head is empty.
+	require.Equal(t, uint64(0), db.Head().NumSeries())
+	require.Equal(t, uint64(0), db.Head().NumStaleSeries())
+
+	// CompactStaleHead should handle zero series gracefully (no panic, no error).
+	require.NoError(t, db.CompactStaleHead())
+
+	// Should still have no blocks since there was nothing to compact.
+	require.Empty(t, db.Blocks())
+}

tsdb/head.go

@@ -1203,6 +1203,36 @@ func (h *Head) truncateMemory(mint int64) (err error) {
 	return h.truncateSeriesAndChunkDiskMapper("truncateMemory")
 }
 
+// truncateStaleSeries removes the provided series as long as they are still stale.
+func (h *Head) truncateStaleSeries(seriesRefs []storage.SeriesRef, maxt int64) error {
+	h.chunkSnapshotMtx.Lock()
+	defer h.chunkSnapshotMtx.Unlock()
+
+	if h.MinTime() >= maxt {
+		return nil
+	}
+
+	h.WaitForPendingReadersInTimeRange(h.MinTime(), maxt)
+
+	deleted := h.gcStaleSeries(seriesRefs, maxt)
+
+	// Record these stale series refs in the WAL so that we can ignore them during replay.
+	if h.wal != nil {
+		stones := make([]tombstones.Stone, 0, len(seriesRefs))
+		for ref := range deleted {
+			stones = append(stones, tombstones.Stone{
+				Ref:       ref,
+				Intervals: tombstones.Intervals{{Mint: math.MinInt64, Maxt: math.MaxInt64}},
+			})
+		}
+		var enc record.Encoder
+		if err := h.wal.Log(enc.Tombstones(stones, nil)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // WaitForPendingReadersInTimeRange waits for queries overlapping with given range to finish querying.
 // The query timeout limits the max wait time of this function implicitly.
 // The mint is inclusive and maxt is the truncation time hence exclusive.
@@ -1556,6 +1586,53 @@ func (h *RangeHead) String() string {
 	return fmt.Sprintf("range head (mint: %d, maxt: %d)", h.MinTime(), h.MaxTime())
 }
 
+// StaleHead allows querying the stale series in the Head via an IndexReader, ChunkReader and tombstones.Reader.
+// Used only for compactions.
+type StaleHead struct {
+	RangeHead
+	staleSeriesRefs []storage.SeriesRef
+}
+
+// NewStaleHead returns a *StaleHead.
+func NewStaleHead(head *Head, mint, maxt int64, staleSeriesRefs []storage.SeriesRef) *StaleHead {
+	return &StaleHead{
+		RangeHead: RangeHead{
+			head: head,
+			mint: mint,
+			maxt: maxt,
+		},
+		staleSeriesRefs: staleSeriesRefs,
+	}
+}
+
+func (h *StaleHead) Index() (_ IndexReader, err error) {
+	return h.head.staleIndex(h.mint, h.maxt, h.staleSeriesRefs)
+}
+
+func (h *StaleHead) NumSeries() uint64 {
+	return h.head.NumStaleSeries()
+}
+
+var staleHeadULID = ulid.MustParse("0000000000XXXXXXXSTALEHEAD")
+
+func (h *StaleHead) Meta() BlockMeta {
+	return BlockMeta{
+		MinTime: h.MinTime(),
+		MaxTime: h.MaxTime(),
+		ULID:    staleHeadULID,
+		Stats: BlockStats{
+			NumSeries: h.NumSeries(),
+		},
+	}
+}
+
+// String returns an human readable representation of the stake head. It's important to
+// keep this function in order to avoid the struct dump when the head is stringified in
+// errors or logs.
+func (h *StaleHead) String() string {
+	return fmt.Sprintf("stale head (mint: %d, maxt: %d)", h.MinTime(), h.MaxTime())
+}
+
 // Delete all samples in the range of [mint, maxt] for series that satisfy the given
 // label matchers.
 func (h *Head) Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Matcher) error {
@@ -1625,13 +1702,14 @@ func (h *Head) gc() (actualInOrderMint, minOOOTime int64, minMmapFile int) {
 
 	// Drop old chunks and remember series IDs and hashes if they can be
 	// deleted entirely.
-	deleted, affected, chunksRemoved, actualInOrderMint, minOOOTime, minMmapFile := h.series.gc(mint, minOOOMmapRef, &h.numStaleSeries)
+	deleted, affected, chunksRemoved, staleSeriesDeleted, actualInOrderMint, minOOOTime, minMmapFile := h.series.gc(mint, minOOOMmapRef)
 	seriesRemoved := len(deleted)
 
 	h.metrics.seriesRemoved.Add(float64(seriesRemoved))
 	h.metrics.chunksRemoved.Add(float64(chunksRemoved))
 	h.metrics.chunks.Sub(float64(chunksRemoved))
 	h.numSeries.Sub(uint64(seriesRemoved))
+	h.numStaleSeries.Sub(uint64(staleSeriesDeleted))
 
 	// Remove deleted series IDs from the postings lists.
 	h.postings.Delete(deleted, affected)
@@ -1948,13 +2026,14 @@ func newStripeSeries(stripeSize int, seriesCallback SeriesLifecycleCallback) *st
 // but the returned map goes into postings.Delete() which expects a map[storage.SeriesRef]struct
 // and there's no easy way to cast maps.
 // minMmapFile is the min mmap file number seen in the series (in-order and out-of-order) after gc'ing the series.
-func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef, numStaleSeries *atomic.Uint64) (_ map[storage.SeriesRef]struct{}, _ map[labels.Label]struct{}, _ int, _, _ int64, minMmapFile int) {
+func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef) (_ map[storage.SeriesRef]struct{}, _ map[labels.Label]struct{}, _, _ int, _, _ int64, minMmapFile int) {
 	var (
-		deleted          = map[storage.SeriesRef]struct{}{}
-		affected         = map[labels.Label]struct{}{}
-		rmChunks         = 0
-		actualMint int64 = math.MaxInt64
-		minOOOTime int64 = math.MaxInt64
+		deleted                  = map[storage.SeriesRef]struct{}{}
+		affected                 = map[labels.Label]struct{}{}
+		rmChunks                 = 0
+		staleSeriesDeleted       = 0
+		actualMint         int64 = math.MaxInt64
+		minOOOTime         int64 = math.MaxInt64
 	)
 	minMmapFile = math.MaxInt32
 
@@ -2009,7 +2088,7 @@ func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef, n
 		if value.IsStaleNaN(series.lastValue) ||
 			(series.lastHistogramValue != nil && value.IsStaleNaN(series.lastHistogramValue.Sum)) ||
 			(series.lastFloatHistogramValue != nil && value.IsStaleNaN(series.lastFloatHistogramValue.Sum)) {
-			numStaleSeries.Dec()
+			staleSeriesDeleted++
 		}
 
 		deleted[storage.SeriesRef(series.ref)] = struct{}{}
@@ -2025,7 +2104,166 @@ func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef, n
 		actualMint = mint
 	}
 
-	return deleted, affected, rmChunks, actualMint, minOOOTime, minMmapFile
+	return deleted, affected, rmChunks, staleSeriesDeleted, actualMint, minOOOTime, minMmapFile
+}
+
+// gcStaleSeries removes all the provided series as long as they are still stale
+// and the series maxt is <= the given max.
+// The returned references are the series that got deleted.
+func (h *Head) gcStaleSeries(seriesRefs []storage.SeriesRef, maxt int64) map[storage.SeriesRef]struct{} {
+	// Drop old chunks and remember series IDs and hashes if they can be
+	// deleted entirely.
+	deleted, affected, chunksRemoved := h.series.gcStaleSeries(seriesRefs, maxt)
+	seriesRemoved := len(deleted)
+
+	h.metrics.seriesRemoved.Add(float64(seriesRemoved))
+	h.metrics.chunksRemoved.Add(float64(chunksRemoved))
+	h.metrics.chunks.Sub(float64(chunksRemoved))
+	h.numSeries.Sub(uint64(seriesRemoved))
+	h.numStaleSeries.Sub(uint64(seriesRemoved))
+
+	// Remove deleted series IDs from the postings lists.
+	h.postings.Delete(deleted, affected)
+
+	// Remove tombstones referring to the deleted series.
+	h.tombstones.DeleteTombstones(deleted)
+
+	if h.wal != nil {
+		_, last, _ := wlog.Segments(h.wal.Dir())
+		h.walExpiriesMtx.Lock()
+		// Keep series records until we're past segment 'last'
+		// because the WAL will still have samples records with
+		// this ref ID. If we didn't keep these series records then
+		// on start up when we replay the WAL, or any other code
+		// that reads the WAL, wouldn't be able to use those
+		// samples since we would have no labels for that ref ID.
+		for ref := range deleted {
+			h.walExpiries[chunks.HeadSeriesRef(ref)] = int64(last)
+		}
+		h.walExpiriesMtx.Unlock()
+	}
+
+	return deleted
+}
+
+// deleteSeriesByID deletes the series with the given reference.
+// Only used for WAL replay.
+func (h *Head) deleteSeriesByID(refs []chunks.HeadSeriesRef) {
+	var (
+		deleted            = map[storage.SeriesRef]struct{}{}
+		affected           = map[labels.Label]struct{}{}
+		staleSeriesDeleted = 0
+		chunksRemoved      = 0
+	)
+
+	for _, ref := range refs {
+		refShard := int(ref) & (h.series.size - 1)
+		h.series.locks[refShard].Lock()
+
+		// Copying getByID here to avoid locking and unlocking twice.
+		series := h.series.series[refShard][ref]
+		if series == nil {
+			h.series.locks[refShard].Unlock()
+			continue
+		}
+
+		if value.IsStaleNaN(series.lastValue) ||
+			(series.lastHistogramValue != nil && value.IsStaleNaN(series.lastHistogramValue.Sum)) ||
+			(series.lastFloatHistogramValue != nil && value.IsStaleNaN(series.lastFloatHistogramValue.Sum)) {
+			staleSeriesDeleted++
+		}
+
+		hash := series.lset.Hash()
+		hashShard := int(hash) & (h.series.size - 1)
+
+		chunksRemoved += len(series.mmappedChunks)
+		if series.headChunks != nil {
+			chunksRemoved += series.headChunks.len()
+		}
+
+		deleted[storage.SeriesRef(series.ref)] = struct{}{}
+		series.lset.Range(func(l labels.Label) { affected[l] = struct{}{} })
+		h.series.hashes[hashShard].del(hash, series.ref)
+		delete(h.series.series[refShard], series.ref)
+
+		h.series.locks[refShard].Unlock()
+	}
+
+	h.metrics.seriesRemoved.Add(float64(len(deleted)))
+	h.metrics.chunksRemoved.Add(float64(chunksRemoved))
+	h.metrics.chunks.Sub(float64(chunksRemoved))
+	h.numSeries.Sub(uint64(len(deleted)))
+	h.numStaleSeries.Sub(uint64(staleSeriesDeleted))
+
+	// Remove deleted series IDs from the postings lists.
+	h.postings.Delete(deleted, affected)
+
+	// Remove tombstones referring to the deleted series.
+	h.tombstones.DeleteTombstones(deleted)
+}
+
+// gcStaleSeries removes all the stale series provided that they are still stale
+// and the series maxt is <= the given max.
+func (s *stripeSeries) gcStaleSeries(seriesRefs []storage.SeriesRef, maxt int64) (_ map[storage.SeriesRef]struct{}, _ map[labels.Label]struct{}, _ int) {
+	var (
+		deleted  = map[storage.SeriesRef]struct{}{}
+		affected = map[labels.Label]struct{}{}
+		rmChunks = 0
+	)
+
+	staleSeriesMap := map[storage.SeriesRef]struct{}{}
+	for _, ref := range seriesRefs {
+		staleSeriesMap[ref] = struct{}{}
+	}
+
+	check := func(hashShard int, hash uint64, series *memSeries, deletedForCallback map[chunks.HeadSeriesRef]labels.Labels) {
+		if _, exists := staleSeriesMap[storage.SeriesRef(series.ref)]; !exists {
+			// This series was not compacted. Skip it.
+			return
+		}
+
+		series.Lock()
+		defer series.Unlock()
+
+		if series.maxTime() > maxt {
+			return
+		}
+
+		// Check if the series is still stale.
+		isStale := value.IsStaleNaN(series.lastValue) ||
+			(series.lastHistogramValue != nil && value.IsStaleNaN(series.lastHistogramValue.Sum)) ||
+			(series.lastFloatHistogramValue != nil && value.IsStaleNaN(series.lastFloatHistogramValue.Sum))
+
+		if !isStale {
+			return
+		}
+
+		if series.headChunks != nil {
+			rmChunks += series.headChunks.len()
+		}
+		rmChunks += len(series.mmappedChunks)
+
+		// The series is gone entirely. We need to keep the series lock
+		// and make sure we have acquired the stripe locks for hash and ID of the
+		// series alike.
+		// If we don't hold them all, there's a very small chance that a series receives
+		// samples again while we are half-way into deleting it.
+		refShard := int(series.ref) & (s.size - 1)
+		if hashShard != refShard {
+			s.locks[refShard].Lock()
+			defer s.locks[refShard].Unlock()
+		}
+
+		deleted[storage.SeriesRef(series.ref)] = struct{}{}
+		series.lset.Range(func(l labels.Label) { affected[l] = struct{}{} })
+		s.hashes[hashShard].del(hash, series.ref)
+		delete(s.series[refShard], series.ref)
+		deletedForCallback[series.ref] = series.lset // OK to access lset; series is locked at the top of this function.
+	}
+
+	s.iterForDeletion(check)
+
+	return deleted, affected, rmChunks
 }
 
 // The iterForDeletion function iterates through all series, invoking the checkDeletedFunc for each.

tsdb/head_append.go

@@ -168,8 +168,6 @@ func (h *Head) appender() *headAppender {
 		headAppenderBase: headAppenderBase{
 			head:                  h,
 			minValidTime:          minValidTime,
-			mint:                  math.MaxInt64,
-			maxt:                  math.MinInt64,
 			headMaxt:              h.MaxTime(),
 			oooTimeWindow:         h.opts.OutOfOrderTimeWindow.Load(),
 			seriesRefs:            h.getRefSeriesBuffer(),
@@ -393,7 +391,6 @@ func (b *appendBatch) close(h *Head) {
 type headAppenderBase struct {
 	head          *Head
 	minValidTime  int64 // No samples below this timestamp are allowed.
-	mint, maxt    int64
 	headMaxt      int64 // We track it here to not take the lock for every sample appended.
 	oooTimeWindow int64 // Use the same for the entire append, and don't load the atomic for each sample.
 
@@ -477,13 +474,6 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64
 		return 0, err
 	}
 
-	if t < a.mint {
-		a.mint = t
-	}
-	if t > a.maxt {
-		a.maxt = t
-	}
-
 	b := a.getCurrentBatch(stFloat, s.ref)
 	b.floats = append(b.floats, record.RefSample{
 		Ref: s.ref,
@@ -527,9 +517,6 @@ func (a *headAppender) AppendSTZeroSample(ref storage.SeriesRef, lset labels.Lab
 		return storage.SeriesRef(s.ref), storage.ErrOutOfOrderST
 	}
 
-	if st > a.maxt {
-		a.maxt = st
-	}
 	b := a.getCurrentBatch(stFloat, s.ref)
 	b.floats = append(b.floats, record.RefSample{Ref: s.ref, T: st, V: 0.0})
 	b.floatSeries = append(b.floatSeries, s)
@@ -903,13 +890,6 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
 		b.floatHistogramSeries = append(b.floatHistogramSeries, s)
 	}
 
-	if t < a.mint {
-		a.mint = t
-	}
-	if t > a.maxt {
-		a.maxt = t
-	}
-
 	return storage.SeriesRef(s.ref), nil
 }
 
@@ -1013,10 +993,6 @@ func (a *headAppender) AppendHistogramSTZeroSample(ref storage.SeriesRef, lset l
 		b.floatHistogramSeries = append(b.floatHistogramSeries, s)
 	}
 
-	if st > a.maxt {
-		a.maxt = st
-	}
-
 	return storage.SeriesRef(s.ref), nil
 }
 

tsdb/head_append_v2.go

@@ -17,7 +17,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"math"
 
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
@@ -89,8 +88,6 @@ func (h *Head) appenderV2() *headAppenderV2 {
 		headAppenderBase: headAppenderBase{
 			head:                  h,
 			minValidTime:          minValidTime,
-			mint:                  math.MaxInt64,
-			maxt:                  math.MinInt64,
 			headMaxt:              h.MaxTime(),
 			oooTimeWindow:         h.opts.OutOfOrderTimeWindow.Load(),
 			seriesRefs:            h.getRefSeriesBuffer(),
@@ -193,13 +190,6 @@ func (a *headAppenderV2) Append(ref storage.SeriesRef, ls labels.Labels, st, t i
 		return 0, appErr
 	}
 
-	if t < a.mint {
-		a.mint = t
-	}
-	if t > a.maxt {
-		a.maxt = t
-	}
-
 	if isStale {
 		// For stale values we never attempt to process metadata/exemplars, claim the success.
 		return storage.SeriesRef(s.ref), nil
@@ -210,9 +200,6 @@ func (a *headAppenderV2) Append(ref storage.SeriesRef, ls labels.Labels, st, t i
 		// Currently only exemplars can return partial errors.
 		partialErr = a.appendExemplars(s, opts.Exemplars)
 	}
-
-	// TODO(bwplotka): Move/reuse metadata tests from scrape, once scrape adopts AppenderV2.
-	// Currently tsdb package does not test metadata.
 	if a.head.opts.EnableMetadataWALRecords && !opts.Metadata.IsEmpty() {
 		s.Lock()
 		metaChanged := s.meta == nil || !s.meta.Equals(opts.Metadata)
@@ -390,10 +377,6 @@ func (a *headAppenderV2) bestEffortAppendSTZeroSample(s *memSeries, ls labels.La
 		a.head.logger.Debug("Error when appending ST", "series", s.lset.String(), "st", st, "t", t, "err", err)
 		return
 	}
-
-	if st > a.maxt {
-		a.maxt = st
-	}
 }
 
 var _ storage.GetRef = &headAppenderV2{}

tsdb/head_append_v2_test.go

@@ -4111,10 +4111,18 @@ func TestHeadAppenderV2_Append_EnableSTAsZeroSample(t *testing.T) {
 	// Make sure counter resets hints are non-zero, so we can detect ST histogram samples.
 	testHistogram := tsdbutil.GenerateTestHistogram(1)
 	testHistogram.CounterResetHint = histogram.NotCounterReset
+
 	testFloatHistogram := tsdbutil.GenerateTestFloatHistogram(1)
 	testFloatHistogram.CounterResetHint = histogram.NotCounterReset
+
+	testNHCB := tsdbutil.GenerateTestCustomBucketsHistogram(1)
+	testNHCB.CounterResetHint = histogram.NotCounterReset
+
+	testFloatNHCB := tsdbutil.GenerateTestCustomBucketsFloatHistogram(1)
+	testFloatNHCB.CounterResetHint = histogram.NotCounterReset
+
 	// TODO(beorn7): Once issue #15346 is fixed, the CounterResetHint of the
-	// following two zero histograms should be histogram.CounterReset.
+	// following zero histograms should be histogram.CounterReset.
 	testZeroHistogram := &histogram.Histogram{
 		Schema:          testHistogram.Schema,
 		ZeroThreshold:   testHistogram.ZeroThreshold,
@@ -4131,6 +4139,19 @@ func TestHeadAppenderV2_Append_EnableSTAsZeroSample(t *testing.T) {
 		PositiveBuckets: []float64{0, 0, 0, 0},
 		NegativeBuckets: []float64{0, 0, 0, 0},
 	}
+	testZeroNHCB := &histogram.Histogram{
+		Schema:          testNHCB.Schema,
+		PositiveSpans:   testNHCB.PositiveSpans,
+		PositiveBuckets: []int64{0, 0, 0, 0},
+		CustomValues:    testNHCB.CustomValues,
+	}
+	testZeroFloatNHCB := &histogram.FloatHistogram{
+		Schema:          testFloatNHCB.Schema,
+		PositiveSpans:   testFloatNHCB.PositiveSpans,
+		PositiveBuckets: []float64{0, 0, 0, 0},
+		CustomValues:    testFloatNHCB.CustomValues,
+	}
+
 	type appendableSamples struct {
 		ts      int64
 		fSample float64
@@ -4183,6 +4204,34 @@ func TestHeadAppenderV2_Append_EnableSTAsZeroSample(t *testing.T) {
 				}
 			}(),
 		},
+		{
+			name: "In order ct+normal sample/NHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, h: testNHCB, st: 1},
+				{ts: 101, h: testNHCB, st: 1},
+			},
+			expectedSamples: func() []chunks.Sample {
+				return []chunks.Sample{
+					sample{t: 1, h: testZeroNHCB},
+					sample{t: 100, h: testNHCB},
+					sample{t: 101, h: testNHCB},
+				}
+			}(),
+		},
+		{
+			name: "In order ct+normal sample/floatNHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, fh: testFloatNHCB, st: 1},
+				{ts: 101, fh: testFloatNHCB, st: 1},
+			},
+			expectedSamples: func() []chunks.Sample {
+				return []chunks.Sample{
+					sample{t: 1, fh: testZeroFloatNHCB},
+					sample{t: 100, fh: testFloatNHCB},
+					sample{t: 101, fh: testFloatNHCB},
+				}
+			}(),
+		},
 		{
 			name: "Consecutive appends with same st ignore st/floatSample",
 			appendableSamples: []appendableSamples{
@@ -4223,6 +4272,34 @@ func TestHeadAppenderV2_Append_EnableSTAsZeroSample(t *testing.T) {
 				}
 			}(),
 		},
+		{
+			name: "Consecutive appends with same st ignore st/NHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, h: testNHCB, st: 1},
+				{ts: 101, h: testNHCB, st: 1},
+			},
+			expectedSamples: func() []chunks.Sample {
+				return []chunks.Sample{
+					sample{t: 1, h: testZeroNHCB},
+					sample{t: 100, h: testNHCB},
+					sample{t: 101, h: testNHCB},
+				}
+			}(),
+		},
+		{
+			name: "Consecutive appends with same st ignore st/floatNHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, fh: testFloatNHCB, st: 1},
+				{ts: 101, fh: testFloatNHCB, st: 1},
+			},
+			expectedSamples: func() []chunks.Sample {
+				return []chunks.Sample{
+					sample{t: 1, fh: testZeroFloatNHCB},
+					sample{t: 100, fh: testFloatNHCB},
+					sample{t: 101, fh: testFloatNHCB},
+				}
+			}(),
+		},
 		{
 			name: "Consecutive appends with newer st do not ignore st/floatSample",
 			appendableSamples: []appendableSamples{
@@ -4262,6 +4339,32 @@ func TestHeadAppenderV2_Append_EnableSTAsZeroSample(t *testing.T) {
 				sample{t: 102, fh: testFloatHistogram},
 			},
 		},
+		{
+			name: "Consecutive appends with newer st do not ignore st/NHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, h: testNHCB, st: 1},
+				{ts: 102, h: testNHCB, st: 101},
+			},
+			expectedSamples: []chunks.Sample{
+				sample{t: 1, h: testZeroNHCB},
+				sample{t: 100, h: testNHCB},
+				sample{t: 101, h: testZeroNHCB},
+				sample{t: 102, h: testNHCB},
+			},
+		},
+		{
+			name: "Consecutive appends with newer st do not ignore st/floatNHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, fh: testFloatNHCB, st: 1},
+				{ts: 102, fh: testFloatNHCB, st: 101},
+			},
+			expectedSamples: []chunks.Sample{
+				sample{t: 1, fh: testZeroFloatNHCB},
+				sample{t: 100, fh: testFloatNHCB},
+				sample{t: 101, fh: testZeroFloatNHCB},
+				sample{t: 102, fh: testFloatNHCB},
+			},
+		},
 		{
 			name: "ST equals to previous sample timestamp is ignored/floatSample",
 			appendableSamples: []appendableSamples{
@@ -4302,6 +4405,34 @@ func TestHeadAppenderV2_Append_EnableSTAsZeroSample(t *testing.T) {
 				}
 			}(),
 		},
+		{
+			name: "ST equals to previous sample timestamp is ignored/NHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, h: testNHCB, st: 1},
+				{ts: 101, h: testNHCB, st: 100},
+			},
+			expectedSamples: func() []chunks.Sample {
+				return []chunks.Sample{
+					sample{t: 1, h: testZeroNHCB},
+					sample{t: 100, h: testNHCB},
+					sample{t: 101, h: testNHCB},
+				}
+			}(),
+		},
+		{
+			name: "ST equals to previous sample timestamp is ignored/floatNHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, fh: testFloatNHCB, st: 1},
+				{ts: 101, fh: testFloatNHCB, st: 100},
+			},
+			expectedSamples: func() []chunks.Sample {
+				return []chunks.Sample{
+					sample{t: 1, fh: testZeroFloatNHCB},
+					sample{t: 100, fh: testFloatNHCB},
+					sample{t: 101, fh: testFloatNHCB},
+				}
+			}(),
+		},
 		{
 			name: "ST lower than minValidTime/float",
 			appendableSamples: []appendableSamples{
@@ -4349,6 +4480,40 @@ func TestHeadAppenderV2_Append_EnableSTAsZeroSample(t *testing.T) {
 				}
 			}(),
 		},
+		{
+			name: "ST lower than minValidTime/NHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, h: testNHCB, st: -1},
+			},
+			// ST results ErrOutOfBounds, but ST append is best effort, so
+			// ST should be ignored, but sample appended.
+			expectedSamples: func() []chunks.Sample {
+				// NOTE: Without ST, on query, first histogram sample will get
+				// CounterReset adjusted to 0.
+				firstSample := testNHCB.Copy()
+				firstSample.CounterResetHint = histogram.UnknownCounterReset
+				return []chunks.Sample{
+					sample{t: 100, h: firstSample},
+				}
+			}(),
+		},
+		{
+			name: "ST lower than minValidTime/floatNHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, fh: testFloatNHCB, st: -1},
+			},
+			// ST results ErrOutOfBounds, but ST append is best effort, so
+			// ST should be ignored, but sample appended.
+			expectedSamples: func() []chunks.Sample {
+				// NOTE: Without ST, on query, first histogram sample will get
+				// CounterReset adjusted to 0.
+				firstSample := testFloatNHCB.Copy()
+				firstSample.CounterResetHint = histogram.UnknownCounterReset
+				return []chunks.Sample{
+					sample{t: 100, fh: firstSample},
+				}
+			}(),
+		},
 		{
 			name: "ST duplicates an existing sample/float",
 			appendableSamples: []appendableSamples{
@@ -4402,6 +4567,44 @@ func TestHeadAppenderV2_Append_EnableSTAsZeroSample(t *testing.T) {
 				}
 			}(),
 		},
+		{
+			name: "ST duplicates an existing sample/NHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, h: testNHCB},
+				{ts: 200, h: testNHCB, st: 100},
+			},
+			// ST results ErrDuplicateSampleForTimestamp, but ST append is best effort, so
+			// ST should be ignored, but sample appended.
+			expectedSamples: func() []chunks.Sample {
+				// NOTE: Without ST, on query, first histogram sample will get
+				// CounterReset adjusted to 0.
+				firstSample := testNHCB.Copy()
+				firstSample.CounterResetHint = histogram.UnknownCounterReset
+				return []chunks.Sample{
+					sample{t: 100, h: firstSample},
+					sample{t: 200, h: testNHCB},
+				}
+			}(),
+		},
+		{
+			name: "ST duplicates an existing sample/floatNHCB",
+			appendableSamples: []appendableSamples{
+				{ts: 100, fh: testFloatNHCB},
+				{ts: 200, fh: testFloatNHCB, st: 100},
+			},
+			// ST results ErrDuplicateSampleForTimestamp, but ST append is best effort, so
+			// ST should ignored, but sample appended.
+			expectedSamples: func() []chunks.Sample {
+				// NOTE: Without ST, on query, first histogram sample will get
+				// CounterReset adjusted to 0.
+				firstSample := testFloatNHCB.Copy()
+				firstSample.CounterResetHint = histogram.UnknownCounterReset
+				return []chunks.Sample{
+					sample{t: 100, fh: firstSample},
+					sample{t: 200, fh: testFloatNHCB},
+				}
+			}(),
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			opts := newTestHeadDefaultOptions(DefaultBlockDuration, false)

tsdb/head_read.go

@@ -22,6 +22,7 @@ import (
 	"sync"
 
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/value"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/chunks"
@@ -201,6 +202,112 @@ func (h *headIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchB
 	return nil
 }
 
+func (h *Head) staleIndex(mint, maxt int64, staleSeriesRefs []storage.SeriesRef) (*headStaleIndexReader, error) {
+	return &headStaleIndexReader{
+		headIndexReader: h.indexRange(mint, maxt),
+		staleSeriesRefs: staleSeriesRefs,
+	}, nil
+}
+
+// headStaleIndexReader gives the stale series that have no out-of-order data.
+// This is only used for stale series compaction at the moment, that will only ask for all
+// the series during compaction. So to make that efficient, this index reader requires the
+// pre-calculated list of stale series refs that can be returned without re-reading the Head.
+type headStaleIndexReader struct {
+	*headIndexReader
+	staleSeriesRefs []storage.SeriesRef
+}
+
+func (h *headStaleIndexReader) Postings(ctx context.Context, name string, values ...string) (index.Postings, error) {
+	// If all postings are requested, return the precalculated list.
+	k, v := index.AllPostingsKey()
+	if len(h.staleSeriesRefs) > 0 && name == k && len(values) == 1 && values[0] == v {
+		return index.NewListPostings(h.staleSeriesRefs), nil
+	}
+	seriesRefs, err := h.head.filterStaleSeriesAndSortPostings(h.head.postings.Postings(ctx, name, values...))
+	if err != nil {
+		return index.ErrPostings(err), err
+	}
+	return index.NewListPostings(seriesRefs), nil
+}
+
+func (h *headStaleIndexReader) PostingsForLabelMatching(ctx context.Context, name string, match func(string) bool) index.Postings {
+	// Unused for compaction, so we don't need to optimise.
+	seriesRefs, err := h.head.filterStaleSeriesAndSortPostings(h.head.postings.PostingsForLabelMatching(ctx, name, match))
+	if err != nil {
+		return index.ErrPostings(err)
+	}
+	return index.NewListPostings(seriesRefs)
+}
+
+func (h *headStaleIndexReader) PostingsForAllLabelValues(ctx context.Context, name string) index.Postings {
+	// Unused for compaction, so we don't need to optimise.
+	seriesRefs, err := h.head.filterStaleSeriesAndSortPostings(h.head.postings.PostingsForAllLabelValues(ctx, name))
+	if err != nil {
+		return index.ErrPostings(err)
+	}
+	return index.NewListPostings(seriesRefs)
+}
+
+// filterStaleSeriesAndSortPostings returns the stale series references from the given postings
+// that also do not have any out-of-order data.
+func (h *Head) filterStaleSeriesAndSortPostings(p index.Postings) ([]storage.SeriesRef, error) {
+	series := make([]*memSeries, 0, 1024)
+
+	notFoundSeriesCount := 0
+	for p.Next() {
+		s := h.series.getByID(chunks.HeadSeriesRef(p.At()))
+		if s == nil {
+			notFoundSeriesCount++
+			continue
+		}
+
+		s.Lock()
+		if s.ooo != nil {
+			// Has out-of-order data; skip it because we cannot determine if a series
+			// is stale when it's getting out-of-order data.
+			s.Unlock()
+			continue
+		}
+
+		if value.IsStaleNaN(s.lastValue) ||
+			(s.lastHistogramValue != nil && value.IsStaleNaN(s.lastHistogramValue.Sum)) ||
+			(s.lastFloatHistogramValue != nil && value.IsStaleNaN(s.lastFloatHistogramValue.Sum)) {
+			series = append(series, s)
+		}
+		s.Unlock()
+	}
+	if notFoundSeriesCount > 0 {
+		h.logger.Debug("Looked up stale series not found", "count", notFoundSeriesCount)
+	}
+	if err := p.Err(); err != nil {
+		return nil, fmt.Errorf("expand postings: %w", err)
+	}
+
+	slices.SortFunc(series, func(a, b *memSeries) int {
+		return labels.Compare(a.labels(), b.labels())
+	})
+
+	refs := make([]storage.SeriesRef, 0, len(series))
+	for _, p := range series {
+		refs = append(refs, storage.SeriesRef(p.ref))
+	}
+	return refs, nil
+}
+
+// SortedPostings returns the postings as it is because we expect any postings obtained via
+// headStaleIndexReader to be already sorted.
+func (*headStaleIndexReader) SortedPostings(p index.Postings) index.Postings {
+	// All the postings function above already give the sorted list of postings.
+	return p
+}
+
+// SortedStaleSeriesRefsNoOOOData returns all the series refs of the stale series that do not have any out-of-order data.
+func (h *Head) SortedStaleSeriesRefsNoOOOData(ctx context.Context) ([]storage.SeriesRef, error) {
+	k, v := index.AllPostingsKey()
+	return h.filterStaleSeriesAndSortPostings(h.postings.Postings(ctx, k, v))
+}
+
 func appendSeriesChunks(s *memSeries, mint, maxt int64, chks []chunks.Meta) []chunks.Meta {
 	for i, c := range s.mmappedChunks {
 		// Do not expose chunks that are outside of the specified range.

tsdb/head_test.go

@@ -6519,7 +6519,7 @@ func TestStripeSeries_gc(t *testing.T) {
 	s, ms1, ms2 := stripeSeriesWithCollidingSeries(t)
 	hash := ms1.lset.Hash()
 
-	s.gc(0, 0, nil)
+	s.gc(0, 0)
 
 	// Verify that we can get neither ms1 nor ms2 after gc-ing corresponding series
 	got := s.getByHash(hash, ms1.lset)

tsdb/head_wal.go

@@ -308,7 +308,21 @@ Outer:
 			}
 			h.wlReplaySamplesPool.Put(v)
 		case []tombstones.Stone:
+			// Tombstone records will be fairly rare, so not trying to optimise the allocations here.
+			deleteSeriesShards := make([][]chunks.HeadSeriesRef, concurrency)
 			for _, s := range v {
+				if len(s.Intervals) == 1 && s.Intervals[0].Mint == math.MinInt64 && s.Intervals[0].Maxt == math.MaxInt64 {
+					// This series was fully deleted at this point. This record is only done for stale series at the moment.
+					mod := uint64(s.Ref) % uint64(concurrency)
+					deleteSeriesShards[mod] = append(deleteSeriesShards[mod], chunks.HeadSeriesRef(s.Ref))
+
+					// If the series is with a different reference, try deleting that.
+					if r, ok := multiRef[chunks.HeadSeriesRef(s.Ref)]; ok {
+						mod := uint64(r) % uint64(concurrency)
+						deleteSeriesShards[mod] = append(deleteSeriesShards[mod], r)
+					}
+					continue
+				}
 				for _, itv := range s.Intervals {
 					if itv.Maxt < h.minValidTime.Load() {
 						continue
@@ -326,6 +340,14 @@ Outer:
 					h.tombstones.AddInterval(s.Ref, itv)
 				}
 			}
+
+			for i := range concurrency {
+				if len(deleteSeriesShards[i]) > 0 {
+					processors[i].input <- walSubsetProcessorInputItem{deletedSeriesRefs: deleteSeriesShards[i]}
+					deleteSeriesShards[i] = nil
+				}
+			}
+
 			h.wlReplaytStonesPool.Put(v)
 		case []record.RefExemplar:
 			for _, e := range v {
@@ -558,10 +580,11 @@ type walSubsetProcessor struct {
 }
 
 type walSubsetProcessorInputItem struct {
-	samples          []record.RefSample
-	histogramSamples []histogramRecord
-	existingSeries   *memSeries
-	walSeriesRef     chunks.HeadSeriesRef
+	samples           []record.RefSample
+	histogramSamples  []histogramRecord
+	existingSeries    *memSeries
+	walSeriesRef      chunks.HeadSeriesRef
+	deletedSeriesRefs []chunks.HeadSeriesRef
 }
 
 func (wp *walSubsetProcessor) setup() {
@@ -712,6 +735,10 @@ func (wp *walSubsetProcessor) processWALSamples(h *Head, mmappedChunks, oooMmapp
 		case wp.histogramsOutput <- in.histogramSamples:
 		default:
 		}
+
+		if len(in.deletedSeriesRefs) > 0 {
+			h.deleteSeriesByID(in.deletedSeriesRefs)
+		}
 	}
 	h.updateMinMaxTime(mint, maxt)
 

util/teststorage/appender.go

@@ -24,7 +24,6 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/atomic"
@@ -97,37 +96,32 @@ func (s Sample) Equals(other Sample) bool {
 		slices.EqualFunc(s.ES, other.ES, exemplar.Exemplar.Equals)
 }
 
-var (
-	sampleComparer = cmp.Comparer(func(a, b Sample) bool {
-		return a.Equals(b)
-	})
-	byLabelSort = cmpopts.SortSlices(func(a, b Sample) int {
-		return labels.Compare(a.L, b.L)
-	})
-)
-
-func includeStaleNaNs(s []Sample) bool {
-	for _, e := range s {
-		if value.IsStaleNaN(e.V) {
-			return true
-		}
+// IsStale returns whether the sample represents a stale sample, according to
+// https://prometheus.io/docs/specs/native_histograms/#staleness-markers.
+func (s Sample) IsStale() bool {
+	switch {
+	case s.FH != nil:
+		return value.IsStaleNaN(s.FH.Sum)
+	case s.H != nil:
+		return value.IsStaleNaN(s.H.Sum)
+	default:
+		return value.IsStaleNaN(s.V)
 	}
-	return false
 }
 
+var sampleComparer = cmp.Comparer(func(a, b Sample) bool {
+	return a.Equals(b)
+})
+
 // RequireEqual is a special require equal that correctly compare Prometheus structures.
 //
 // In comparison to testutil.RequireEqual, this function adds special logic for comparing []Samples.
 //
-// It also ignores ordering when expected slice contains at least one StaleNaN. This is because the
-// scrape StaleNan samples are generated by iterating over a map, thus expectedly different.
-//
-// TODO(bwplotka): We should likely reorder only within a group of sequential NaNs or only in scrape package.
+// It also ignores ordering between consecutive stale samples to avoid false
+// negatives due to map iteration order in staleness tracking.
 func RequireEqual(t testing.TB, expected, got []Sample, msgAndArgs ...any) {
 	opts := []cmp.Option{sampleComparer}
-	if includeStaleNaNs(expected) {
-		opts = append(opts, byLabelSort)
-	}
+	expected = reorderExpectedForStaleness(expected, got)
 	testutil.RequireEqualWithOptions(t, expected, got, opts, msgAndArgs...)
 }
 
@@ -136,9 +130,7 @@ func RequireNotEqual(t testing.TB, expected, got []Sample, msgAndArgs ...any) {
 	t.Helper()
 
 	opts := []cmp.Option{cmp.Comparer(labels.Equal), sampleComparer}
-	if includeStaleNaNs(expected) {
-		opts = append(opts, byLabelSort)
-	}
+	expected = reorderExpectedForStaleness(expected, got)
 	if !cmp.Equal(expected, got, opts...) {
 		return
 	}
@@ -147,6 +139,45 @@ func RequireNotEqual(t testing.TB, expected, got []Sample, msgAndArgs ...any) {
 		"b: %s", expected, got), msgAndArgs...)
 }
 
+func reorderExpectedForStaleness(expected, got []Sample) []Sample {
+	if len(expected) != len(got) || !includeStaleNaNs(expected) {
+		return expected
+	}
+	result := make([]Sample, len(expected))
+	copy(result, expected)
+
+	// Try to reorder only consecutive stale samples to avoid false negatives
+	// due to map iteration order in staleness tracking.
+	for i := range result {
+		if !result[i].IsStale() {
+			continue
+		}
+		if result[i].Equals(got[i]) {
+			continue
+		}
+		for j := i + 1; j < len(result); j++ {
+			if !result[j].IsStale() {
+				break
+			}
+			if result[j].Equals(got[i]) {
+				// Swap.
+				result[i], result[j] = result[j], result[i]
+				break
+			}
+		}
+	}
+	return result
+}
+
+func includeStaleNaNs(s []Sample) bool {
+	for _, e := range s {
+		if e.IsStale() {
+			return true
+		}
+	}
+	return false
+}
+
 // Appendable is a storage.Appendable mock.
 // It allows recording all samples that were added through the appender and injecting errors.
 // Appendable will panic if more than one Appender is open.

util/teststorage/appender_test.go

@@ -306,3 +306,108 @@ func TestConcurrentAppenderV2_ReturnsErrAppender(t *testing.T) {
 	require.Error(t, app.Commit())
 	require.Error(t, app.Rollback())
 }
+
+func TestReorderExpectedForStaleness(t *testing.T) {
+	testcases := []struct {
+		name       string
+		inExpected []Sample
+		inGot      []Sample
+		expected   []Sample
+	}{
+		{
+			name: "no staleness markers",
+			inExpected: []Sample{
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+				{L: labels.FromStrings("a", "2"), T: 1, V: 2},
+			},
+			inGot: []Sample{
+				{L: labels.FromStrings("a", "2"), T: 1, V: 2},
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+			},
+		},
+		{
+			name: "with staleness markers",
+			inExpected: []Sample{
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+				{L: labels.FromStrings("a", "2"), T: 2, V: 2},
+				{L: labels.FromStrings("a", "3"), T: 3, V: math.Float64frombits(value.StaleNaN)},
+				{L: labels.FromStrings("a", "4"), T: 4, V: math.Float64frombits(value.StaleNaN)},
+			},
+			inGot: []Sample{
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+				{L: labels.FromStrings("a", "2"), T: 2, V: 2},
+				{L: labels.FromStrings("a", "3"), T: 3, V: math.Float64frombits(value.StaleNaN)},
+				{L: labels.FromStrings("a", "4"), T: 4, V: math.Float64frombits(value.StaleNaN)},
+			},
+		},
+		{
+			name: "with staleness markers wrong order",
+			inExpected: []Sample{
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+				{L: labels.FromStrings("a", "2"), T: 2, V: 2},
+				{L: labels.FromStrings("a", "3"), T: 3, V: math.Float64frombits(value.StaleNaN)},
+				{L: labels.FromStrings("a", "4"), T: 4, V: math.Float64frombits(value.StaleNaN)},
+			},
+			inGot: []Sample{
+				{L: labels.FromStrings("a", "2"), T: 2, V: 2},
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+				{L: labels.FromStrings("a", "4"), T: 4, V: math.Float64frombits(value.StaleNaN)},
+				{L: labels.FromStrings("a", "3"), T: 3, V: math.Float64frombits(value.StaleNaN)},
+			},
+			expected: []Sample{
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+				{L: labels.FromStrings("a", "2"), T: 2, V: 2},
+				{L: labels.FromStrings("a", "4"), T: 4, V: math.Float64frombits(value.StaleNaN)},
+				{L: labels.FromStrings("a", "3"), T: 3, V: math.Float64frombits(value.StaleNaN)},
+			},
+		},
+		{
+			name: "with staleness markers wrong order but not consecutive",
+			inExpected: []Sample{
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+				{L: labels.FromStrings("a", "3"), T: 3, V: math.Float64frombits(value.StaleNaN)},
+				{L: labels.FromStrings("a", "2"), T: 2, V: 2},
+				{L: labels.FromStrings("a", "4"), T: 4, V: math.Float64frombits(value.StaleNaN)},
+			},
+			inGot: []Sample{
+				{L: labels.FromStrings("a", "2"), T: 2, V: 2},
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+				{L: labels.FromStrings("a", "4"), T: 4, V: math.Float64frombits(value.StaleNaN)},
+				{L: labels.FromStrings("a", "3"), T: 3, V: math.Float64frombits(value.StaleNaN)},
+			},
+			expected: []Sample{
+				{L: labels.FromStrings("a", "1"), T: 1, V: 1},
+				{L: labels.FromStrings("a", "3"), T: 3, V: math.Float64frombits(value.StaleNaN)},
+				{L: labels.FromStrings("a", "2"), T: 2, V: 2},
+				{L: labels.FromStrings("a", "4"), T: 4, V: math.Float64frombits(value.StaleNaN)},
+			},
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.expected == nil {
+				tc.expected = tc.inExpected
+			}
+			RequireEqual(t, tc.expected, reorderExpectedForStaleness(tc.inExpected, tc.inGot))
+		})
+	}
+}
+
+func TestSampleIsStale(t *testing.T) {
+	s1 := Sample{V: 1}
+	require.False(t, s1.IsStale())
+	s2 := Sample{V: math.Float64frombits(value.StaleNaN)}
+	require.True(t, s2.IsStale())
+	h := tsdbutil.GenerateTestHistogram(0)
+	h1 := Sample{V: math.Float64frombits(value.StaleNaN), H: h}
+	require.False(t, h1.IsStale()) // Histogram takes precedence over V.
+	h.Sum = math.Float64frombits(value.StaleNaN)
+	h2 := Sample{V: 1, H: h}
+	require.True(t, h2.IsStale())
+	fh := tsdbutil.GenerateTestFloatHistogram(0)
+	fh1 := Sample{V: math.Float64frombits(value.StaleNaN), H: h, FH: fh}
+	require.False(t, fh1.IsStale()) // FloatHistogram takes precedence over all.
+	fh.Sum = math.Float64frombits(value.StaleNaN)
+	fh2 := Sample{V: 1, H: tsdbutil.GenerateTestHistogram(1), FH: fh}
+	require.True(t, fh2.IsStale())
+}

util/teststorage/storage.go

@@ -19,12 +19,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 
-	"github.com/prometheus/prometheus/model/exemplar"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
 )
 
@@ -32,14 +28,22 @@ type Option func(opt *tsdb.Options)
 
 // New returns a new TestStorage for testing purposes
 // that removes all associated files on closing.
+//
+// Caller does not need to close the TestStorage after use, it's deferred via t.Cleanup.
 func New(t testing.TB, o ...Option) *TestStorage {
 	s, err := NewWithError(o...)
 	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		_ = s.Close() // Ignore errors, as it could be a double close.
+	})
 	return s
 }
 
 // NewWithError returns a new TestStorage for user facing tests, which reports
 // errors directly.
+//
+// It's a caller responsibility to close the TestStorage after use.
 func NewWithError(o ...Option) (*TestStorage, error) {
 	// Tests just load data for a series sequentially. Thus we
 	// need a long appendable window.
@@ -49,6 +53,10 @@ func NewWithError(o ...Option) (*TestStorage, error) {
 	opts.RetentionDuration = 0
 	opts.OutOfOrderTimeWindow = 0
 
+	// Enable exemplars storage by default.
+	opts.EnableExemplarStorage = true
+	opts.MaxExemplars = 1e5
+
 	for _, opt := range o {
 		opt(opts)
 	}
@@ -62,20 +70,12 @@ func NewWithError(o ...Option) (*TestStorage, error) {
 	if err != nil {
 		return nil, fmt.Errorf("opening test storage: %w", err)
 	}
-	reg := prometheus.NewRegistry()
-	eMetrics := tsdb.NewExemplarMetrics(reg)
-
-	es, err := tsdb.NewCircularExemplarStorage(10, eMetrics, opts.OutOfOrderTimeWindow)
-	if err != nil {
-		return nil, fmt.Errorf("opening test exemplar storage: %w", err)
-	}
-	return &TestStorage{DB: db, exemplarStorage: es, dir: dir}, nil
+	return &TestStorage{DB: db, dir: dir}, nil
 }
 
 type TestStorage struct {
 	*tsdb.DB
-	exemplarStorage tsdb.ExemplarStorage
-	dir             string
+	dir string
 }
 
 func (s TestStorage) Close() error {
@@ -84,15 +84,3 @@ func (s TestStorage) Close() error {
 	}
 	return os.RemoveAll(s.dir)
 }
-
-func (s TestStorage) ExemplarAppender() storage.ExemplarAppender {
-	return s
-}
-
-func (s TestStorage) ExemplarQueryable() storage.ExemplarQueryable {
-	return s.exemplarStorage
-}
-
-func (s TestStorage) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) {
-	return ref, s.exemplarStorage.AddExemplar(l, e)
-}

web/api/testhelpers/api.go

@@ -0,0 +1,244 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package testhelpers provides utilities for testing the Prometheus HTTP API.
+// This file contains helper functions for creating test API instances and managing test lifecycles.
+package testhelpers
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/promslog"
+
+	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/promql/promqltest"
+	"github.com/prometheus/prometheus/rules"
+	"github.com/prometheus/prometheus/scrape"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/tsdb"
+	"github.com/prometheus/prometheus/util/notifications"
+)
+
+// RulesRetriever provides a list of active rules and alerts.
+type RulesRetriever interface {
+	RuleGroups() []*rules.Group
+	AlertingRules() []*rules.AlertingRule
+}
+
+// TargetRetriever provides the list of active/dropped targets to scrape or not.
+type TargetRetriever interface {
+	TargetsActive() map[string][]*scrape.Target
+	TargetsDropped() map[string][]*scrape.Target
+	TargetsDroppedCounts() map[string]int
+	ScrapePoolConfig(string) (*config.ScrapeConfig, error)
+}
+
+// ScrapePoolsRetriever provide the list of all scrape pools.
+type ScrapePoolsRetriever interface {
+	ScrapePools() []string
+}
+
+// AlertmanagerRetriever provides a list of all/dropped AlertManager URLs.
+type AlertmanagerRetriever interface {
+	Alertmanagers() []*url.URL
+	DroppedAlertmanagers() []*url.URL
+}
+
+// TSDBAdminStats provides TSDB admin statistics.
+type TSDBAdminStats interface {
+	CleanTombstones() error
+	Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Matcher) error
+	Snapshot(dir string, withHead bool) error
+	Stats(statsByLabelName string, limit int) (*tsdb.Stats, error)
+	WALReplayStatus() (tsdb.WALReplayStatus, error)
+	BlockMetas() ([]tsdb.BlockMeta, error)
+}
+
+// APIConfig holds configuration for creating a test API instance.
+type APIConfig struct {
+	// Core dependencies.
+	QueryEngine       *LazyLoader[promql.QueryEngine]
+	Queryable         *LazyLoader[storage.SampleAndChunkQueryable]
+	ExemplarQueryable *LazyLoader[storage.ExemplarQueryable]
+
+	// Retrievers.
+	RulesRetriever        *LazyLoader[RulesRetriever]
+	TargetRetriever       *LazyLoader[TargetRetriever]
+	ScrapePoolsRetriever  *LazyLoader[ScrapePoolsRetriever]
+	AlertmanagerRetriever *LazyLoader[AlertmanagerRetriever]
+
+	// Admin.
+	TSDBAdmin *LazyLoader[TSDBAdminStats]
+	DBDir     string
+
+	// Optional overrides.
+	Config   func() config.Config
+	FlagsMap map[string]string
+	Now      func() time.Time
+}
+
+// APIWrapper wraps the API and provides a handler for testing.
+type APIWrapper struct {
+	Handler http.Handler
+}
+
+// PrometheusVersion contains build information about Prometheus.
+type PrometheusVersion struct {
+	Version   string `json:"version"`
+	Revision  string `json:"revision"`
+	Branch    string `json:"branch"`
+	BuildUser string `json:"buildUser"`
+	BuildDate string `json:"buildDate"`
+	GoVersion string `json:"goVersion"`
+}
+
+// RuntimeInfo contains runtime information about Prometheus.
+type RuntimeInfo struct {
+	StartTime           time.Time `json:"startTime"`
+	CWD                 string    `json:"CWD"`
+	Hostname            string    `json:"hostname"`
+	ServerTime          time.Time `json:"serverTime"`
+	ReloadConfigSuccess bool      `json:"reloadConfigSuccess"`
+	LastConfigTime      time.Time `json:"lastConfigTime"`
+	CorruptionCount     int64     `json:"corruptionCount"`
+	GoroutineCount      int       `json:"goroutineCount"`
+	GOMAXPROCS          int       `json:"GOMAXPROCS"`
+	GOMEMLIMIT          int64     `json:"GOMEMLIMIT"`
+	GOGC                string    `json:"GOGC"`
+	GODEBUG             string    `json:"GODEBUG"`
+	StorageRetention    string    `json:"storageRetention"`
+}
+
+// NewAPIParams holds all the parameters needed to create a v1.API instance.
+type NewAPIParams struct {
+	QueryEngine           promql.QueryEngine
+	Queryable             storage.SampleAndChunkQueryable
+	ExemplarQueryable     storage.ExemplarQueryable
+	ScrapePoolsRetriever  func(context.Context) ScrapePoolsRetriever
+	TargetRetriever       func(context.Context) TargetRetriever
+	AlertmanagerRetriever func(context.Context) AlertmanagerRetriever
+	ConfigFunc            func() config.Config
+	FlagsMap              map[string]string
+	ReadyFunc             func(http.HandlerFunc) http.HandlerFunc
+	TSDBAdmin             TSDBAdminStats
+	DBDir                 string
+	Logger                *slog.Logger
+	RulesRetriever        func(context.Context) RulesRetriever
+	RuntimeInfoFunc       func() (RuntimeInfo, error)
+	BuildInfo             *PrometheusVersion
+	NotificationsGetter   func() []notifications.Notification
+	NotificationsSub      func() (<-chan notifications.Notification, func(), bool)
+	Gatherer              prometheus.Gatherer
+	Registerer            prometheus.Registerer
+}
+
+// PrepareAPI creates a NewAPIParams with sensible defaults for testing.
+func PrepareAPI(t *testing.T, cfg APIConfig) NewAPIParams {
+	t.Helper()
+
+	// Create defaults for unset lazy loaders.
+	if cfg.QueryEngine == nil {
+		cfg.QueryEngine = NewLazyLoader(func() promql.QueryEngine {
+			return promqltest.NewTestEngineWithOpts(t, promql.EngineOpts{
+				Logger:                   nil,
+				Reg:                      nil,
+				MaxSamples:               10000,
+				Timeout:                  100 * time.Second,
+				NoStepSubqueryIntervalFn: func(int64) int64 { return 60 * 1000 },
+				EnableAtModifier:         true,
+				EnableNegativeOffset:     true,
+				EnablePerStepStats:       true,
+			})
+		})
+	}
+
+	if cfg.Queryable == nil {
+		cfg.Queryable = NewLazyLoader(NewEmptyQueryable)
+	}
+
+	if cfg.ExemplarQueryable == nil {
+		cfg.ExemplarQueryable = NewLazyLoader(NewEmptyExemplarQueryable)
+	}
+
+	if cfg.RulesRetriever == nil {
+		cfg.RulesRetriever = NewLazyLoader(func() RulesRetriever {
+			return NewEmptyRulesRetriever()
+		})
+	}
+
+	if cfg.TargetRetriever == nil {
+		cfg.TargetRetriever = NewLazyLoader(func() TargetRetriever {
+			return NewEmptyTargetRetriever()
+		})
+	}
+
+	if cfg.ScrapePoolsRetriever == nil {
+		cfg.ScrapePoolsRetriever = NewLazyLoader(func() ScrapePoolsRetriever {
+			return NewEmptyScrapePoolsRetriever()
+		})
+	}
+
+	if cfg.AlertmanagerRetriever == nil {
+		cfg.AlertmanagerRetriever = NewLazyLoader(func() AlertmanagerRetriever {
+			return NewEmptyAlertmanagerRetriever()
+		})
+	}
+
+	if cfg.TSDBAdmin == nil {
+		cfg.TSDBAdmin = NewLazyLoader(func() TSDBAdminStats {
+			return NewEmptyTSDBAdminStats()
+		})
+	}
+
+	if cfg.Config == nil {
+		cfg.Config = func() config.Config { return config.Config{} }
+	}
+
+	if cfg.FlagsMap == nil {
+		cfg.FlagsMap = map[string]string{}
+	}
+
+	if cfg.DBDir == "" {
+		cfg.DBDir = t.TempDir()
+	}
+
+	return NewAPIParams{
+		QueryEngine:           cfg.QueryEngine.Get(),
+		Queryable:             cfg.Queryable.Get(),
+		ExemplarQueryable:     cfg.ExemplarQueryable.Get(),
+		ScrapePoolsRetriever:  func(context.Context) ScrapePoolsRetriever { return cfg.ScrapePoolsRetriever.Get() },
+		TargetRetriever:       func(context.Context) TargetRetriever { return cfg.TargetRetriever.Get() },
+		AlertmanagerRetriever: func(context.Context) AlertmanagerRetriever { return cfg.AlertmanagerRetriever.Get() },
+		ConfigFunc:            cfg.Config,
+		FlagsMap:              cfg.FlagsMap,
+		ReadyFunc:             func(f http.HandlerFunc) http.HandlerFunc { return f },
+		TSDBAdmin:             cfg.TSDBAdmin.Get(),
+		DBDir:                 cfg.DBDir,
+		Logger:                promslog.NewNopLogger(),
+		RulesRetriever:        func(context.Context) RulesRetriever { return cfg.RulesRetriever.Get() },
+		RuntimeInfoFunc:       func() (RuntimeInfo, error) { return RuntimeInfo{}, nil },
+		BuildInfo:             &PrometheusVersion{},
+		NotificationsGetter:   func() []notifications.Notification { return nil },
+		NotificationsSub:      func() (<-chan notifications.Notification, func(), bool) { return nil, func() {}, false },
+		Gatherer:              prometheus.NewRegistry(),
+		Registerer:            prometheus.NewRegistry(),
+	}
+}

web/api/testhelpers/assertions.go

@@ -0,0 +1,252 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file provides assertion helpers for validating API responses in tests.
+package testhelpers
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/stretchr/testify/require"
+)
+
+// RequireSuccess asserts that the response has status "success" and returns the response for chaining.
+func (r *Response) RequireSuccess() *Response {
+	r.t.Helper()
+	require.NotNil(r.t, r.JSON, "response body is not JSON")
+	require.Equal(r.t, "success", r.JSON["status"], "expected status to be 'success'")
+	return r
+}
+
+// RequireError asserts that the response has status "error" and returns the response for chaining.
+func (r *Response) RequireError() *Response {
+	r.t.Helper()
+	require.NotNil(r.t, r.JSON, "response body is not JSON")
+	require.Equal(r.t, "error", r.JSON["status"], "expected status to be 'error'")
+	return r
+}
+
+// RequireStatusCode asserts that the response has the given HTTP status code and returns the response for chaining.
+func (r *Response) RequireStatusCode(expectedCode int) *Response {
+	r.t.Helper()
+	require.Equal(r.t, expectedCode, r.StatusCode, "unexpected HTTP status code")
+	return r
+}
+
+// RequireJSONPathExists asserts that a JSON path exists and returns the response for chaining.
+func (r *Response) RequireJSONPathExists(path string) *Response {
+	r.t.Helper()
+	require.NotNil(r.t, r.JSON, "response body is not JSON")
+
+	value := getJSONPath(r.JSON, path)
+	require.NotNil(r.t, value, "JSON path %q does not exist", path)
+	return r
+}
+
+// RequireEquals asserts that a JSON path equals the expected value and returns the response for chaining.
+func (r *Response) RequireEquals(path string, expected any) *Response {
+	r.t.Helper()
+	require.NotNil(r.t, r.JSON, "response body is not JSON")
+
+	value := getJSONPath(r.JSON, path)
+	require.NotNil(r.t, value, "JSON path %q does not exist", path)
+	require.Equal(r.t, expected, value, "JSON path %q has unexpected value", path)
+	return r
+}
+
+// RequireJSONArray asserts that a JSON path contains an array and returns the response for chaining.
+func (r *Response) RequireJSONArray(path string) *Response {
+	r.t.Helper()
+	require.NotNil(r.t, r.JSON, "response body is not JSON")
+
+	value := getJSONPath(r.JSON, path)
+	require.NotNil(r.t, value, "JSON path %q does not exist", path)
+	_, ok := value.([]any)
+	require.True(r.t, ok, "JSON path %q is not an array", path)
+	return r
+}
+
+// RequireLenAtLeast asserts that a JSON path contains an array with at least minLen elements and returns the response for chaining.
+func (r *Response) RequireLenAtLeast(path string, minLen int) *Response {
+	r.t.Helper()
+	require.NotNil(r.t, r.JSON, "response body is not JSON")
+
+	value := getJSONPath(r.JSON, path)
+	require.NotNil(r.t, value, "JSON path %q does not exist", path)
+	arr, ok := value.([]any)
+	require.True(r.t, ok, "JSON path %q is not an array", path)
+	require.GreaterOrEqual(r.t, len(arr), minLen, "JSON path %q has fewer than %d elements", path, minLen)
+	return r
+}
+
+// RequireArrayContains asserts that a JSON path contains an array with the expected element and returns the response for chaining.
+func (r *Response) RequireArrayContains(path string, expected any) *Response {
+	r.t.Helper()
+	require.NotNil(r.t, r.JSON, "response body is not JSON")
+
+	value := getJSONPath(r.JSON, path)
+	require.NotNil(r.t, value, "JSON path %q does not exist", path)
+	arr, ok := value.([]any)
+	require.True(r.t, ok, "JSON path %q is not an array", path)
+
+	found := slices.Contains(arr, expected)
+	require.True(r.t, found, "JSON path %q does not contain expected value %v", path, expected)
+	return r
+}
+
+// RequireSome asserts that at least one element in an array satisfies the predicate and returns the response for chaining.
+func (r *Response) RequireSome(path string, predicate func(any) bool) *Response {
+	r.t.Helper()
+	require.NotNil(r.t, r.JSON, "response body is not JSON")
+
+	value := getJSONPath(r.JSON, path)
+	require.NotNil(r.t, value, "JSON path %q does not exist", path)
+	arr, ok := value.([]any)
+	require.True(r.t, ok, "JSON path %q is not an array", path)
+
+	found := slices.ContainsFunc(arr, predicate)
+	require.True(r.t, found, "no element in JSON path %q satisfies the predicate", path)
+	return r
+}
+
+// getJSONPath extracts a value from a JSON object using a simple path notation.
+// Supports paths like "$.data", "$.data.groups", "$.data.groups[0]".
+func getJSONPath(data map[string]any, path string) any {
+	// Remove leading "$." if present.
+	path = strings.TrimPrefix(path, "$.")
+
+	if path == "" {
+		return data
+	}
+
+	parts := strings.Split(path, ".")
+	current := any(data)
+
+	for _, part := range parts {
+		// Handle array indexing (e.g., "groups[0]").
+		if strings.Contains(part, "[") {
+			// Not implementing array indexing for simplicity.
+			// Tests should use direct field access or RequireSome.
+			return nil
+		}
+
+		// Navigate to the next level.
+		m, ok := current.(map[string]any)
+		if !ok {
+			return nil
+		}
+		current = m[part]
+	}
+
+	return current
+}
+
+// RequireVectorResult is a convenience helper for checking vector query results.
+func (r *Response) RequireVectorResult() *Response {
+	r.t.Helper()
+	return r.RequireSuccess().RequireEquals("$.data.resultType", "vector")
+}
+
+// RequireMatrixResult is a convenience helper for checking matrix query results.
+func (r *Response) RequireMatrixResult() *Response {
+	r.t.Helper()
+	return r.RequireSuccess().RequireEquals("$.data.resultType", "matrix")
+}
+
+// RequireScalarResult is a convenience helper for checking scalar query results.
+func (r *Response) RequireScalarResult() *Response {
+	r.t.Helper()
+	return r.RequireSuccess().RequireEquals("$.data.resultType", "scalar")
+}
+
+// RequireRulesGroupNamed asserts that a rules response contains a group with the given name.
+func (r *Response) RequireRulesGroupNamed(name string) *Response {
+	r.t.Helper()
+	return r.RequireSuccess().RequireSome("$.data.groups", func(group any) bool {
+		if g, ok := group.(map[string]any); ok {
+			return g["name"] == name
+		}
+		return false
+	})
+}
+
+// RequireTargetCount asserts that a targets response contains at least n targets.
+func (r *Response) RequireTargetCount(minCount int) *Response {
+	r.t.Helper()
+	r.RequireSuccess()
+
+	// The targets endpoint returns activeTargets as an array of targets.
+	value := getJSONPath(r.JSON, "$.data.activeTargets")
+	require.NotNil(r.t, value, "JSON path $.data.activeTargets does not exist")
+
+	arr, ok := value.([]any)
+	require.True(r.t, ok, "$.data.activeTargets is not an array")
+	require.GreaterOrEqual(r.t, len(arr), minCount, "expected at least %d targets, got %d", minCount, len(arr))
+	return r
+}
+
+// DebugJSON is a helper for debugging JSON responses in tests.
+func (r *Response) DebugJSON() *Response {
+	r.t.Helper()
+	r.t.Logf("Response status code: %d", r.StatusCode)
+	r.t.Logf("Response body: %s", r.Body)
+	if r.JSON != nil {
+		r.t.Logf("Response JSON: %+v", r.JSON)
+	}
+	return r
+}
+
+// RequireContainsSubstring asserts that the response body contains the given substring.
+func (r *Response) RequireContainsSubstring(substring string) *Response {
+	r.t.Helper()
+	require.Contains(r.t, r.Body, substring, "response body does not contain expected substring")
+	return r
+}
+
+// RequireField asserts that a field exists at the given path and returns its value.
+// Note: This method cannot be chained further since it returns the field value, not the Response.
+func (r *Response) RequireField(path string) any {
+	r.t.Helper()
+	require.NotNil(r.t, r.JSON, "response body is not JSON")
+
+	value := getJSONPath(r.JSON, path)
+	require.NotNil(r.t, value, "JSON path %q does not exist", path)
+	return value
+}
+
+// RequireFieldType asserts that a field exists and has the expected type.
+func (r *Response) RequireFieldType(path, expectedType string) *Response {
+	r.t.Helper()
+	value := r.RequireField(path)
+
+	var actualType string
+	switch value.(type) {
+	case string:
+		actualType = "string"
+	case float64:
+		actualType = "number"
+	case bool:
+		actualType = "bool"
+	case []any:
+		actualType = "array"
+	case map[string]any:
+		actualType = "object"
+	default:
+		actualType = fmt.Sprintf("%T", value)
+	}
+
+	require.Equal(r.t, expectedType, actualType, "JSON path %q has unexpected type", path)
+	return r
+}

web/api/testhelpers/fixtures.go

@@ -0,0 +1,178 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file provides test fixture data for API tests.
+package testhelpers
+
+import (
+	"time"
+
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/promql/parser"
+	"github.com/prometheus/prometheus/rules"
+	"github.com/prometheus/prometheus/storage"
+)
+
+// FixtureSeries creates a simple series with the "up" metric.
+func FixtureSeries() []storage.Series {
+	// Use timestamps relative to "now" so queries work.
+	now := time.Now().UnixMilli()
+	return []storage.Series{
+		&FakeSeries{
+			labels: labels.FromStrings("__name__", "up", "job", "prometheus", "instance", "localhost:9090"),
+			samples: []promql.FPoint{
+				{T: now - 120000, F: 1},
+				{T: now - 60000, F: 1},
+				{T: now, F: 1},
+			},
+		},
+	}
+}
+
+// FixtureMultipleSeries creates multiple series for testing.
+func FixtureMultipleSeries() []storage.Series {
+	// Use timestamps relative to "now" so queries work.
+	now := time.Now().UnixMilli()
+	return []storage.Series{
+		&FakeSeries{
+			labels: labels.FromStrings("__name__", "up", "job", "prometheus", "instance", "localhost:9090"),
+			samples: []promql.FPoint{
+				{T: now - 60000, F: 1},
+				{T: now, F: 1},
+			},
+		},
+		&FakeSeries{
+			labels: labels.FromStrings("__name__", "up", "job", "node", "instance", "localhost:9100"),
+			samples: []promql.FPoint{
+				{T: now - 60000, F: 1},
+				{T: now, F: 0},
+			},
+		},
+		&FakeSeries{
+			labels: labels.FromStrings("__name__", "http_requests_total", "job", "api", "instance", "localhost:8080"),
+			samples: []promql.FPoint{
+				{T: now - 60000, F: 100},
+				{T: now, F: 150},
+			},
+		},
+	}
+}
+
+// FixtureRuleGroups creates a simple set of rule groups for testing.
+func FixtureRuleGroups() []*rules.Group {
+	// Create a simple recording rule.
+	expr, _ := parser.ParseExpr("up == 1")
+	recordingRule := rules.NewRecordingRule(
+		"job:up:sum",
+		expr,
+		labels.EmptyLabels(),
+	)
+
+	// Create a simple alerting rule.
+	alertExpr, _ := parser.ParseExpr("up == 0")
+	alertingRule := rules.NewAlertingRule(
+		"InstanceDown",
+		alertExpr,
+		time.Minute,
+		0,
+		labels.FromStrings("severity", "critical"),
+		labels.EmptyLabels(),
+		labels.EmptyLabels(),
+		"Instance {{ $labels.instance }} is down",
+		true,
+		nil,
+	)
+
+	// Create a rule group.
+	group := rules.NewGroup(rules.GroupOptions{
+		Name:     "example",
+		File:     "example.rules",
+		Interval: time.Minute,
+		Rules: []rules.Rule{
+			recordingRule,
+			alertingRule,
+		},
+	})
+
+	return []*rules.Group{group}
+}
+
+// FixtureEmptyRuleGroups returns an empty set of rule groups.
+func FixtureEmptyRuleGroups() []*rules.Group {
+	return []*rules.Group{}
+}
+
+// FixtureSingleSeries creates a single series for simple tests.
+func FixtureSingleSeries(metricName string, value float64) []storage.Series {
+	return []storage.Series{
+		&FakeSeries{
+			labels: labels.FromStrings("__name__", metricName),
+			samples: []promql.FPoint{
+				{T: 0, F: value},
+			},
+		},
+	}
+}
+
+// FixtureHistogramSeries creates a series with native histogram data.
+func FixtureHistogramSeries() []storage.Series {
+	// Use timestamps relative to "now" so queries work.
+	now := time.Now().UnixMilli()
+	return []storage.Series{
+		&FakeHistogramSeries{
+			labels: labels.FromStrings("__name__", "test_histogram", "job", "prometheus", "instance", "localhost:9090"),
+			histograms: []promql.HPoint{
+				{
+					T: now - 60000,
+					H: &histogram.FloatHistogram{
+						Schema:        2,
+						ZeroThreshold: 0.001,
+						ZeroCount:     5,
+						Count:         50,
+						Sum:           100,
+						PositiveSpans: []histogram.Span{
+							{Offset: 0, Length: 2},
+							{Offset: 1, Length: 2},
+						},
+						NegativeSpans: []histogram.Span{
+							{Offset: 0, Length: 1},
+						},
+						PositiveBuckets: []float64{5, 10, 8, 7},
+						NegativeBuckets: []float64{3},
+					},
+				},
+				{
+					T: now,
+					H: &histogram.FloatHistogram{
+						Schema:        2,
+						ZeroThreshold: 0.001,
+						ZeroCount:     8,
+						Count:         60,
+						Sum:           120,
+						PositiveSpans: []histogram.Span{
+							{Offset: 0, Length: 2},
+							{Offset: 1, Length: 2},
+						},
+						NegativeSpans: []histogram.Span{
+							{Offset: 0, Length: 1},
+						},
+						PositiveBuckets: []float64{6, 12, 10, 9},
+						NegativeBuckets: []float64{4},
+					},
+				},
+			},
+		},
+	}
+}

web/api/testhelpers/mocks.go

@@ -0,0 +1,534 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file contains mock implementations of API dependencies for testing.
+package testhelpers
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/model/exemplar"
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/rules"
+	"github.com/prometheus/prometheus/scrape"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/tsdb"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+	"github.com/prometheus/prometheus/tsdb/chunks"
+	"github.com/prometheus/prometheus/util/annotations"
+)
+
+// LazyLoader allows lazy initialization of mocks per test.
+type LazyLoader[T any] struct {
+	loader func() T
+	value  *T
+}
+
+// NewLazyLoader creates a new LazyLoader with the given loader function.
+func NewLazyLoader[T any](loader func() T) *LazyLoader[T] {
+	return &LazyLoader[T]{loader: loader}
+}
+
+// Get returns the loaded value, initializing it if necessary.
+func (l *LazyLoader[T]) Get() T {
+	if l.value == nil {
+		v := l.loader()
+		l.value = &v
+	}
+	return *l.value
+}
+
+// FakeQueryable implements storage.SampleAndChunkQueryable with configurable behavior.
+type FakeQueryable struct {
+	series []storage.Series
+}
+
+func (f *FakeQueryable) Querier(_, _ int64) (storage.Querier, error) {
+	return &FakeQuerier{series: f.series}, nil
+}
+
+func (f *FakeQueryable) ChunkQuerier(_, _ int64) (storage.ChunkQuerier, error) {
+	return &FakeChunkQuerier{series: f.series}, nil
+}
+
+// FakeQuerier implements storage.Querier.
+type FakeQuerier struct {
+	series []storage.Series
+}
+
+func (f *FakeQuerier) Select(_ context.Context, _ bool, _ *storage.SelectHints, _ ...*labels.Matcher) storage.SeriesSet {
+	return &FakeSeriesSet{series: f.series, idx: -1}
+}
+
+func (f *FakeQuerier) LabelValues(_ context.Context, name string, _ *storage.LabelHints, _ ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	valuesMap := make(map[string]struct{})
+	for _, s := range f.series {
+		lbls := s.Labels()
+		if val := lbls.Get(name); val != "" {
+			valuesMap[val] = struct{}{}
+		}
+	}
+	values := make([]string, 0, len(valuesMap))
+	for v := range valuesMap {
+		values = append(values, v)
+	}
+	return values, nil, nil
+}
+
+func (f *FakeQuerier) LabelNames(_ context.Context, _ *storage.LabelHints, _ ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	namesMap := make(map[string]struct{})
+	for _, s := range f.series {
+		lbls := s.Labels()
+		lbls.Range(func(l labels.Label) {
+			namesMap[l.Name] = struct{}{}
+		})
+	}
+	names := make([]string, 0, len(namesMap))
+	for n := range namesMap {
+		names = append(names, n)
+	}
+	return names, nil, nil
+}
+
+func (*FakeQuerier) Close() error {
+	return nil
+}
+
+// FakeChunkQuerier implements storage.ChunkQuerier.
+type FakeChunkQuerier struct {
+	series []storage.Series
+}
+
+func (f *FakeChunkQuerier) Select(_ context.Context, _ bool, _ *storage.SelectHints, _ ...*labels.Matcher) storage.ChunkSeriesSet {
+	return &FakeChunkSeriesSet{series: f.series, idx: -1}
+}
+
+func (f *FakeChunkQuerier) LabelValues(_ context.Context, name string, _ *storage.LabelHints, _ ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	valuesMap := make(map[string]struct{})
+	for _, s := range f.series {
+		lbls := s.Labels()
+		if val := lbls.Get(name); val != "" {
+			valuesMap[val] = struct{}{}
+		}
+	}
+	values := make([]string, 0, len(valuesMap))
+	for v := range valuesMap {
+		values = append(values, v)
+	}
+	return values, nil, nil
+}
+
+func (f *FakeChunkQuerier) LabelNames(_ context.Context, _ *storage.LabelHints, _ ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	namesMap := make(map[string]struct{})
+	for _, s := range f.series {
+		lbls := s.Labels()
+		lbls.Range(func(l labels.Label) {
+			namesMap[l.Name] = struct{}{}
+		})
+	}
+	names := make([]string, 0, len(namesMap))
+	for n := range namesMap {
+		names = append(names, n)
+	}
+	return names, nil, nil
+}
+
+func (*FakeChunkQuerier) Close() error {
+	return nil
+}
+
+// FakeSeriesSet implements storage.SeriesSet.
+type FakeSeriesSet struct {
+	series []storage.Series
+	idx    int
+}
+
+func (f *FakeSeriesSet) Next() bool {
+	f.idx++
+	return f.idx < len(f.series)
+}
+
+func (f *FakeSeriesSet) At() storage.Series {
+	return f.series[f.idx]
+}
+
+func (*FakeSeriesSet) Err() error {
+	return nil
+}
+
+func (*FakeSeriesSet) Warnings() annotations.Annotations {
+	return nil
+}
+
+// FakeChunkSeriesSet implements storage.ChunkSeriesSet.
+type FakeChunkSeriesSet struct {
+	series []storage.Series
+	idx    int
+}
+
+func (f *FakeChunkSeriesSet) Next() bool {
+	f.idx++
+	return f.idx < len(f.series)
+}
+
+func (f *FakeChunkSeriesSet) At() storage.ChunkSeries {
+	return &FakeChunkSeries{series: f.series[f.idx]}
+}
+
+func (*FakeChunkSeriesSet) Err() error {
+	return nil
+}
+
+func (*FakeChunkSeriesSet) Warnings() annotations.Annotations {
+	return nil
+}
+
+// FakeChunkSeries implements storage.ChunkSeries.
+type FakeChunkSeries struct {
+	series storage.Series
+}
+
+func (f *FakeChunkSeries) Labels() labels.Labels {
+	return f.series.Labels()
+}
+
+func (*FakeChunkSeries) Iterator(_ chunks.Iterator) chunks.Iterator {
+	return &FakeChunkSeriesIterator{}
+}
+
+// FakeChunkSeriesIterator implements chunks.Iterator.
+type FakeChunkSeriesIterator struct{}
+
+func (*FakeChunkSeriesIterator) Next() bool {
+	return false
+}
+
+func (*FakeChunkSeriesIterator) At() chunks.Meta {
+	return chunks.Meta{}
+}
+
+func (*FakeChunkSeriesIterator) Err() error {
+	return nil
+}
+
+// FakeSeries implements storage.Series.
+type FakeSeries struct {
+	labels  labels.Labels
+	samples []promql.FPoint
+}
+
+func (f *FakeSeries) Labels() labels.Labels {
+	return f.labels
+}
+
+func (f *FakeSeries) Iterator(chunkenc.Iterator) chunkenc.Iterator {
+	return &FakeSeriesIterator{samples: f.samples, idx: -1}
+}
+
+// FakeSeriesIterator implements chunkenc.Iterator.
+type FakeSeriesIterator struct {
+	samples []promql.FPoint
+	idx     int
+}
+
+func (f *FakeSeriesIterator) Next() chunkenc.ValueType {
+	f.idx++
+	if f.idx < len(f.samples) {
+		return chunkenc.ValFloat
+	}
+	return chunkenc.ValNone
+}
+
+func (f *FakeSeriesIterator) Seek(t int64) chunkenc.ValueType {
+	for f.idx < len(f.samples)-1 {
+		f.idx++
+		if f.samples[f.idx].T >= t {
+			return chunkenc.ValFloat
+		}
+	}
+	return chunkenc.ValNone
+}
+
+func (f *FakeSeriesIterator) At() (int64, float64) {
+	s := f.samples[f.idx]
+	return s.T, s.F
+}
+
+func (*FakeSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
+	panic("not implemented")
+}
+
+func (*FakeSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
+	panic("not implemented")
+}
+
+func (f *FakeSeriesIterator) AtT() int64 {
+	return f.samples[f.idx].T
+}
+
+func (*FakeSeriesIterator) AtST() int64 {
+	return 0
+}
+
+func (*FakeSeriesIterator) Err() error {
+	return nil
+}
+
+// FakeHistogramSeries implements storage.Series for histogram data.
+type FakeHistogramSeries struct {
+	labels     labels.Labels
+	histograms []promql.HPoint
+}
+
+func (f *FakeHistogramSeries) Labels() labels.Labels {
+	return f.labels
+}
+
+func (f *FakeHistogramSeries) Iterator(chunkenc.Iterator) chunkenc.Iterator {
+	return &FakeHistogramSeriesIterator{histograms: f.histograms, idx: -1}
+}
+
+// FakeHistogramSeriesIterator implements chunkenc.Iterator for histogram data.
+type FakeHistogramSeriesIterator struct {
+	histograms []promql.HPoint
+	idx        int
+}
+
+func (f *FakeHistogramSeriesIterator) Next() chunkenc.ValueType {
+	f.idx++
+	if f.idx < len(f.histograms) {
+		return chunkenc.ValFloatHistogram
+	}
+	return chunkenc.ValNone
+}
+
+func (f *FakeHistogramSeriesIterator) Seek(t int64) chunkenc.ValueType {
+	for f.idx < len(f.histograms)-1 {
+		f.idx++
+		if f.histograms[f.idx].T >= t {
+			return chunkenc.ValFloatHistogram
+		}
+	}
+	return chunkenc.ValNone
+}
+
+func (*FakeHistogramSeriesIterator) At() (int64, float64) {
+	panic("not a float value")
+}
+
+func (*FakeHistogramSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
+	panic("not implemented")
+}
+
+func (f *FakeHistogramSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
+	h := f.histograms[f.idx]
+	return h.T, h.H
+}
+
+func (f *FakeHistogramSeriesIterator) AtT() int64 {
+	return f.histograms[f.idx].T
+}
+
+func (*FakeHistogramSeriesIterator) AtST() int64 {
+	return 0
+}
+
+func (*FakeHistogramSeriesIterator) Err() error {
+	return nil
+}
+
+// FakeExemplarQueryable implements storage.ExemplarQueryable.
+type FakeExemplarQueryable struct{}
+
+func (*FakeExemplarQueryable) ExemplarQuerier(_ context.Context) (storage.ExemplarQuerier, error) {
+	return &FakeExemplarQuerier{}, nil
+}
+
+// FakeExemplarQuerier implements storage.ExemplarQuerier.
+type FakeExemplarQuerier struct{}
+
+func (*FakeExemplarQuerier) Select(_, _ int64, _ ...[]*labels.Matcher) ([]exemplar.QueryResult, error) {
+	return nil, nil
+}
+
+// FakeRulesRetriever implements v1.RulesRetriever.
+type FakeRulesRetriever struct {
+	groups []*rules.Group
+}
+
+func (f *FakeRulesRetriever) RuleGroups() []*rules.Group {
+	return f.groups
+}
+
+func (f *FakeRulesRetriever) AlertingRules() []*rules.AlertingRule {
+	var alertingRules []*rules.AlertingRule
+	for _, g := range f.groups {
+		for _, r := range g.Rules() {
+			if ar, ok := r.(*rules.AlertingRule); ok {
+				alertingRules = append(alertingRules, ar)
+			}
+		}
+	}
+	return alertingRules
+}
+
+// FakeTargetRetriever implements v1.TargetRetriever.
+type FakeTargetRetriever struct {
+	active        map[string][]*scrape.Target
+	dropped       map[string][]*scrape.Target
+	droppedCounts map[string]int
+	scrapeConfig  map[string]*config.ScrapeConfig
+}
+
+func (f *FakeTargetRetriever) TargetsActive() map[string][]*scrape.Target {
+	if f.active == nil {
+		return make(map[string][]*scrape.Target)
+	}
+	return f.active
+}
+
+func (f *FakeTargetRetriever) TargetsDropped() map[string][]*scrape.Target {
+	if f.dropped == nil {
+		return make(map[string][]*scrape.Target)
+	}
+	return f.dropped
+}
+
+func (f *FakeTargetRetriever) TargetsDroppedCounts() map[string]int {
+	if f.droppedCounts == nil {
+		return make(map[string]int)
+	}
+	return f.droppedCounts
+}
+
+func (f *FakeTargetRetriever) ScrapePoolConfig(name string) (*config.ScrapeConfig, error) {
+	if f.scrapeConfig == nil {
+		return nil, nil
+	}
+	return f.scrapeConfig[name], nil
+}
+
+// FakeScrapePoolsRetriever implements v1.ScrapePoolsRetriever.
+type FakeScrapePoolsRetriever struct {
+	pools []string
+}
+
+func (f *FakeScrapePoolsRetriever) ScrapePools() []string {
+	if f.pools == nil {
+		return []string{}
+	}
+	return f.pools
+}
+
+// FakeAlertmanagerRetriever implements v1.AlertmanagerRetriever.
+type FakeAlertmanagerRetriever struct{}
+
+func (*FakeAlertmanagerRetriever) Alertmanagers() []*url.URL {
+	return nil
+}
+
+func (*FakeAlertmanagerRetriever) DroppedAlertmanagers() []*url.URL {
+	return nil
+}
+
+// FakeTSDBAdminStats implements v1.TSDBAdminStats.
+type FakeTSDBAdminStats struct{}
+
+func (*FakeTSDBAdminStats) CleanTombstones() error {
+	return nil
+}
+
+func (*FakeTSDBAdminStats) Delete(_ context.Context, _, _ int64, _ ...*labels.Matcher) error {
+	return nil
+}
+
+func (*FakeTSDBAdminStats) Snapshot(_ string, _ bool) error {
+	return nil
+}
+
+func (*FakeTSDBAdminStats) Stats(_ string, _ int) (*tsdb.Stats, error) {
+	return &tsdb.Stats{}, nil
+}
+
+func (*FakeTSDBAdminStats) WALReplayStatus() (tsdb.WALReplayStatus, error) {
+	return tsdb.WALReplayStatus{}, nil
+}
+
+func (*FakeTSDBAdminStats) BlockMetas() ([]tsdb.BlockMeta, error) {
+	return []tsdb.BlockMeta{}, nil
+}
+
+// NewEmptyQueryable returns a queryable with no series.
+func NewEmptyQueryable() storage.SampleAndChunkQueryable {
+	return &FakeQueryable{series: []storage.Series{}}
+}
+
+// NewQueryableWithSeries returns a queryable with the given series.
+func NewQueryableWithSeries(series []storage.Series) storage.SampleAndChunkQueryable {
+	return &FakeQueryable{series: series}
+}
+
+// TSDBNotReadyQueryable implements storage.SampleAndChunkQueryable that returns tsdb.ErrNotReady.
+type TSDBNotReadyQueryable struct{}
+
+func (*TSDBNotReadyQueryable) Querier(_, _ int64) (storage.Querier, error) {
+	return nil, tsdb.ErrNotReady
+}
+
+func (*TSDBNotReadyQueryable) ChunkQuerier(_, _ int64) (storage.ChunkQuerier, error) {
+	return nil, tsdb.ErrNotReady
+}
+
+// NewTSDBNotReadyQueryable returns a queryable that always returns tsdb.ErrNotReady.
+func NewTSDBNotReadyQueryable() storage.SampleAndChunkQueryable {
+	return &TSDBNotReadyQueryable{}
+}
+
+// NewEmptyExemplarQueryable returns an exemplar queryable with no exemplars.
+func NewEmptyExemplarQueryable() storage.ExemplarQueryable {
+	return &FakeExemplarQueryable{}
+}
+
+// NewEmptyRulesRetriever returns a rules retriever with no rules.
+func NewEmptyRulesRetriever() *FakeRulesRetriever {
+	return &FakeRulesRetriever{groups: []*rules.Group{}}
+}
+
+// NewRulesRetrieverWithGroups returns a rules retriever with the given groups.
+func NewRulesRetrieverWithGroups(groups []*rules.Group) *FakeRulesRetriever {
+	return &FakeRulesRetriever{groups: groups}
+}
+
+// NewEmptyTargetRetriever returns a target retriever with no targets.
+func NewEmptyTargetRetriever() *FakeTargetRetriever {
+	return &FakeTargetRetriever{}
+}
+
+// NewEmptyScrapePoolsRetriever returns a scrape pools retriever with no pools.
+func NewEmptyScrapePoolsRetriever() *FakeScrapePoolsRetriever {
+	return &FakeScrapePoolsRetriever{pools: []string{}}
+}
+
+// NewEmptyAlertmanagerRetriever returns an alertmanager retriever with no alertmanagers.
+func NewEmptyAlertmanagerRetriever() *FakeAlertmanagerRetriever {
+	return &FakeAlertmanagerRetriever{}
+}
+
+// NewEmptyTSDBAdminStats returns a TSDB admin stats with no-op implementations.
+func NewEmptyTSDBAdminStats() *FakeTSDBAdminStats {
+	return &FakeTSDBAdminStats{}
+}

web/api/testhelpers/openapi.go

@@ -0,0 +1,204 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file provides OpenAPI-specific test utilities for validating spec compliance.
+package testhelpers
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/pb33f/libopenapi"
+	validator "github.com/pb33f/libopenapi-validator"
+	valerrors "github.com/pb33f/libopenapi-validator/errors"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	openAPIValidator31   validator.Validator
+	openAPIValidator32   validator.Validator
+	openAPIValidatorOnce sync.Once
+	openAPIValidatorErr  error
+)
+
+// loadOpenAPIValidators loads and caches both OpenAPI 3.1 and 3.2 validators from golden files.
+func loadOpenAPIValidators() (v31, v32 validator.Validator, err error) {
+	openAPIValidatorOnce.Do(func() {
+		// Load OpenAPI 3.1 validator.
+		goldenPath31 := filepath.Join("testdata", "openapi_3.1_golden.yaml")
+		specBytes31, err := os.ReadFile(goldenPath31)
+		if err != nil {
+			openAPIValidatorErr = fmt.Errorf("failed to read OpenAPI 3.1 spec from %s: %w", goldenPath31, err)
+			return
+		}
+
+		doc31, err := libopenapi.NewDocument(specBytes31)
+		if err != nil {
+			openAPIValidatorErr = fmt.Errorf("failed to parse OpenAPI 3.1 document: %w", err)
+			return
+		}
+
+		v31, errs := validator.NewValidator(doc31)
+		if len(errs) > 0 {
+			openAPIValidatorErr = fmt.Errorf("failed to create OpenAPI 3.1 validator: %v", errs)
+			return
+		}
+
+		openAPIValidator31 = v31
+
+		// Load OpenAPI 3.2 validator.
+		goldenPath32 := filepath.Join("testdata", "openapi_3.2_golden.yaml")
+		specBytes32, err := os.ReadFile(goldenPath32)
+		if err != nil {
+			openAPIValidatorErr = fmt.Errorf("failed to read OpenAPI 3.2 spec from %s: %w", goldenPath32, err)
+			return
+		}
+
+		doc32, err := libopenapi.NewDocument(specBytes32)
+		if err != nil {
+			openAPIValidatorErr = fmt.Errorf("failed to parse OpenAPI 3.2 document: %w", err)
+			return
+		}
+
+		v32, errs := validator.NewValidator(doc32)
+		if len(errs) > 0 {
+			openAPIValidatorErr = fmt.Errorf("failed to create OpenAPI 3.2 validator: %v", errs)
+			return
+		}
+
+		openAPIValidator32 = v32
+	})
+
+	if openAPIValidatorErr != nil {
+		return nil, nil, openAPIValidatorErr
+	}
+
+	return openAPIValidator31, openAPIValidator32, nil
+}
+
+// ValidateOpenAPI validates the request and response against both OpenAPI 3.1 and 3.2 specifications.
+// This ensures API endpoints are compatible with both OpenAPI versions.
+// Returns the response for chaining.
+func (r *Response) ValidateOpenAPI() *Response {
+	r.t.Helper()
+
+	// Load both validators (cached after first call).
+	v31, v32, err := loadOpenAPIValidators()
+	require.NoError(r.t, err, "failed to load OpenAPI validators")
+
+	// Validate against OpenAPI 3.1 spec.
+	if r.request != nil {
+		r.validateRequestWithVersion(v31, "3.1")
+	}
+	r.validateResponseWithVersion(v31, "3.1")
+
+	// Validate against OpenAPI 3.2 spec.
+	if r.request != nil {
+		r.validateRequestWithVersion(v32, "3.2")
+	}
+	r.validateResponseWithVersion(v32, "3.2")
+
+	return r
+}
+
+// validateRequestWithVersion validates the HTTP request against a specific OpenAPI version's spec.
+func (r *Response) validateRequestWithVersion(v validator.Validator, version string) {
+	r.t.Helper()
+
+	// Create a validation request from the original request.
+	validationReq := &http.Request{
+		Method: r.request.Method,
+		URL:    r.request.URL,
+		Header: r.request.Header,
+		Body:   io.NopCloser(bytes.NewReader(r.requestBody)),
+	}
+
+	// Validate the request.
+	valid, errors := v.ValidateHttpRequest(validationReq)
+	if !valid {
+		// Check if the error is because the path doesn't exist in this version.
+		// Some endpoints (like /notifications/live) only exist in 3.2, not 3.1.
+		if isPathNotFoundError(errors) && version == "3.1" && strings.Contains(r.request.URL.Path, "/notifications/live") {
+			// Expected: /notifications/live is only in OpenAPI 3.2.
+			return
+		}
+
+		var errorMessages []string
+		for _, e := range errors {
+			errorMessages = append(errorMessages, e.Error())
+		}
+		require.Fail(r.t, fmt.Sprintf("OpenAPI %s request validation failed", version),
+			"Request to %s %s failed OpenAPI %s validation:\n%v",
+			r.request.Method, r.request.URL.Path, version, errorMessages)
+	}
+}
+
+// validateResponseWithVersion validates the HTTP response against a specific OpenAPI version's spec.
+func (r *Response) validateResponseWithVersion(v validator.Validator, version string) {
+	r.t.Helper()
+
+	// Create a validation request (needed for response validation context).
+	validationReq := &http.Request{
+		Method: r.request.Method,
+		URL:    r.request.URL,
+		Header: r.request.Header,
+	}
+
+	// Create a response for validation.
+	validationResp := &http.Response{
+		StatusCode: r.StatusCode,
+		Header:     r.responseHeader,
+		Body:       io.NopCloser(bytes.NewReader([]byte(r.Body))),
+		Request:    validationReq,
+	}
+
+	// Validate the response.
+	valid, errors := v.ValidateHttpResponse(validationReq, validationResp)
+	if !valid {
+		// Check if the error is because the path doesn't exist in this version.
+		// Some endpoints (like /notifications/live) only exist in 3.2, not 3.1.
+		if isPathNotFoundError(errors) && version == "3.1" && strings.Contains(r.request.URL.Path, "/notifications/live") {
+			// Expected: /notifications/live is only in OpenAPI 3.2.
+			return
+		}
+
+		var errorMessages []string
+		for _, e := range errors {
+			errorMessages = append(errorMessages, e.Error())
+		}
+		require.Fail(r.t, fmt.Sprintf("OpenAPI %s response validation failed", version),
+			"Response from %s %s (status %d) failed OpenAPI %s validation:\n%v",
+			r.request.Method, r.request.URL.Path, r.StatusCode, version, errorMessages)
+	}
+}
+
+// isPathNotFoundError checks if the validation errors indicate a path was not found in the spec.
+func isPathNotFoundError(errors []*valerrors.ValidationError) bool {
+	for _, err := range errors {
+		errStr := err.Error()
+		// Check for common "path not found" error messages from libopenapi-validator.
+		if strings.Contains(errStr, "path") && (strings.Contains(errStr, "not found") || strings.Contains(errStr, "does not exist")) {
+			return true
+		}
+		if strings.Contains(errStr, "GET /notifications/live") || strings.Contains(errStr, "/notifications/live not found") {
+			return true
+		}
+	}
+	return false
+}

web/api/testhelpers/request.go

@@ -0,0 +1,145 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file provides HTTP request builders for testing API endpoints.
+package testhelpers
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+// Response wraps an HTTP response with parsed JSON data.
+// It supports method chaining for assertions.
+//
+// Example usage:
+//
+//	testhelpers.GET(t, api, "/api/v1/query", "query", "up").
+//		ValidateOpenAPI().
+//		RequireSuccess().
+//		RequireEquals("$.data.resultType", "vector").
+//		RequireLenAtLeast("$.data.result", 1)
+//
+//	testhelpers.POST(t, api, "/api/v1/query", "query", "up").
+//		ValidateOpenAPI().
+//		RequireSuccess().
+//		RequireArrayContains("$.data.result", expectedValue)
+type Response struct {
+	StatusCode     int
+	Body           string
+	JSON           map[string]any
+	t              *testing.T
+	request        *http.Request
+	requestBody    []byte
+	responseHeader http.Header
+}
+
+// GET sends a GET request to the API and returns a Response with parsed JSON.
+// queryParams should be pairs of key-value strings.
+func GET(t *testing.T, api *APIWrapper, path string, queryParams ...string) *Response {
+	t.Helper()
+
+	if len(queryParams)%2 != 0 {
+		t.Fatal("queryParams must be key-value pairs")
+	}
+
+	// Build query string.
+	values := url.Values{}
+	for i := 0; i < len(queryParams); i += 2 {
+		values.Add(queryParams[i], queryParams[i+1])
+	}
+
+	fullPath := path
+	if len(values) > 0 {
+		fullPath = path + "?" + values.Encode()
+	}
+
+	req := httptest.NewRequest(http.MethodGet, fullPath, nil)
+	return executeRequest(t, api, req)
+}
+
+// POST sends a POST request to the API with the given body and returns a Response with parsed JSON.
+// bodyParams should be pairs of key-value strings for form data.
+func POST(t *testing.T, api *APIWrapper, path string, bodyParams ...string) *Response {
+	t.Helper()
+
+	if len(bodyParams)%2 != 0 {
+		t.Fatal("bodyParams must be key-value pairs")
+	}
+
+	// Build form data.
+	values := url.Values{}
+	for i := 0; i < len(bodyParams); i += 2 {
+		values.Add(bodyParams[i], bodyParams[i+1])
+	}
+
+	req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(values.Encode()))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	return executeRequest(t, api, req)
+}
+
+// executeRequest executes an HTTP request and parses the response as JSON.
+func executeRequest(t *testing.T, api *APIWrapper, req *http.Request) *Response {
+	t.Helper()
+
+	// Capture the request body for validation.
+	var requestBody []byte
+	if req.Body != nil {
+		var err error
+		requestBody, err = io.ReadAll(req.Body)
+		if err != nil {
+			t.Fatalf("failed to read request body: %v", err)
+		}
+		// Restore the body for the actual request.
+		req.Body = io.NopCloser(strings.NewReader(string(requestBody)))
+	}
+
+	recorder := httptest.NewRecorder()
+	api.Handler.ServeHTTP(recorder, req)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+
+	bodyBytes, err := io.ReadAll(result.Body)
+	if err != nil {
+		t.Fatalf("failed to read response body: %v", err)
+	}
+
+	resp := &Response{
+		StatusCode:     result.StatusCode,
+		Body:           string(bodyBytes),
+		t:              t,
+		request:        req,
+		requestBody:    requestBody,
+		responseHeader: result.Header,
+	}
+
+	// Try to parse as JSON.
+	if result.Header.Get("Content-Type") == "application/json" || strings.Contains(result.Header.Get("Content-Type"), "application/json") {
+		var jsonData map[string]any
+		if err := json.Unmarshal(bodyBytes, &jsonData); err != nil {
+			// If JSON parsing fails, leave JSON as nil.
+			// This allows tests to handle non-JSON responses.
+			resp.JSON = nil
+		} else {
+			resp.JSON = jsonData
+		}
+	}
+
+	return resp
+}

web/api/v1/api.go

@@ -258,6 +258,7 @@ type API struct {
 	codecs []Codec
 
 	featureRegistry features.Collector
+	openAPIBuilder  *OpenAPIBuilder
 }
 
 // NewAPI returns an initialized API type.
@@ -299,6 +300,7 @@ func NewAPI(
 	appendMetadata bool,
 	overrideErrorCode OverrideErrorCode,
 	featureRegistry features.Collector,
+	openAPIOptions OpenAPIOptions,
 ) *API {
 	a := &API{
 		QueryEngine:       qe,
@@ -329,6 +331,7 @@ func NewAPI(
 		notificationsSub:    notificationsSub,
 		overrideErrorCode:   overrideErrorCode,
 		featureRegistry:     featureRegistry,
+		openAPIBuilder:      NewOpenAPIBuilder(openAPIOptions, logger),
 
 		remoteReadHandler: remote.NewReadHandler(logger, registerer, q, configFunc, remoteReadSampleLimit, remoteReadConcurrencyLimit, remoteReadMaxBytesInFrame),
 	}
@@ -400,7 +403,7 @@ func (api *API) Register(r *route.Router) {
 			w.WriteHeader(http.StatusNoContent)
 		})
 		return api.ready(httputil.CompressionHandler{
-			Handler: hf,
+			Handler: api.openAPIBuilder.WrapHandler(hf),
 		}.ServeHTTP)
 	}
 
@@ -469,6 +472,9 @@ func (api *API) Register(r *route.Router) {
 	r.Put("/admin/tsdb/delete_series", wrapAgent(api.deleteSeries))
 	r.Put("/admin/tsdb/clean_tombstones", wrapAgent(api.cleanTombstones))
 	r.Put("/admin/tsdb/snapshot", wrapAgent(api.snapshot))
+
+	// OpenAPI endpoint.
+	r.Get("/openapi.yaml", api.ready(api.openAPIBuilder.ServeOpenAPI))
 }
 
 type QueryData struct {
@@ -1346,13 +1352,19 @@ func (api *API) targetRelabelSteps(r *http.Request) apiFuncResult {
 
 	rules := scrapeConfig.RelabelConfigs
 	steps := make([]RelabelStep, len(rules))
+	lb := labels.NewBuilder(lbls)
+	keep := true
 	for i, rule := range rules {
-		outLabels, keep := relabel.Process(lbls, rules[:i+1]...)
-		steps[i] = RelabelStep{
-			Rule:   rule,
-			Output: outLabels,
-			Keep:   keep,
+		if keep {
+			keep = relabel.ProcessBuilder(lb, rule)
 		}
+
+		outLabels := labels.EmptyLabels()
+		if keep {
+			outLabels = lb.Labels()
+		}
+
+		steps[i] = RelabelStep{Rule: rule, Output: outLabels, Keep: keep}
 	}
 
 	return apiFuncResult{&RelabelStepsResponse{Steps: steps}, nil, nil, nil}

web/api/v1/api_scenarios_test.go

@@ -0,0 +1,419 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/web/api/testhelpers"
+)
+
+// TODO: Generate automated tests from OpenAPI spec to validate API responses.
+
+// TestAPIEmpty tests the API with no metrics and no rules.
+func TestAPIEmpty(t *testing.T) {
+	// Create an API with empty defaults (no series, no rules).
+	api := newTestAPI(t, testhelpers.APIConfig{})
+
+	t.Run("GET /api/v1/labels returns success with empty array", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/labels").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONArray("$.data")
+	})
+
+	t.Run("GET /api/v1/query?query=up returns success (empty result ok)", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/query", "query", "up").
+			ValidateOpenAPI().
+			RequireSuccess().
+			RequireEquals("$.data.resultType", "vector")
+	})
+
+	t.Run("GET /api/v1/query_range?query=up returns success", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/query_range",
+			"query", "up",
+			"start", "0",
+			"end", "100",
+			"step", "10").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "matrix")
+	})
+
+	t.Run("GET /api/v1/series returns success with empty result", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/series",
+			"match[]", "up",
+			"start", "0",
+			"end", "100").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONArray("$.data")
+	})
+
+	t.Run("GET /api/v1/label/__name__/values returns success with empty array", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/label/__name__/values").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONArray("$.data")
+	})
+
+	t.Run("GET /api/v1/targets returns success", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/targets").
+			RequireSuccess().
+			RequireJSONPathExists("$.data.activeTargets")
+	})
+
+	t.Run("GET /api/v1/rules returns success with empty groups", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/rules").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data.groups")
+	})
+
+	t.Run("GET /api/v1/alerts returns success with empty alerts", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/alerts").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data.alerts")
+	})
+
+	t.Run("GET /api/v1/alertmanagers returns success", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/alertmanagers").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data.activeAlertmanagers")
+	})
+
+	t.Run("GET /api/v1/metadata returns success", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/metadata").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data")
+	})
+
+	t.Run("GET /api/v1/status/config returns success", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/status/config").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data.yaml")
+	})
+
+	t.Run("GET /api/v1/status/flags returns success", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/status/flags").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data")
+	})
+
+	t.Run("GET /api/v1/status/runtimeinfo returns success", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/status/runtimeinfo").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data")
+	})
+
+	t.Run("GET /api/v1/status/buildinfo returns success", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/status/buildinfo").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data")
+	})
+
+	t.Run("POST /api/v1/query with form data returns success", func(t *testing.T) {
+		testhelpers.POST(t, api, "/api/v1/query", "query", "up").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "vector")
+	})
+}
+
+// TestAPIWithSeries tests the API with metrics/series data.
+func TestAPIWithSeries(t *testing.T) {
+	// Create an API with sample series data.
+	api := newTestAPI(t, testhelpers.APIConfig{
+		Queryable: testhelpers.NewLazyLoader(func() storage.SampleAndChunkQueryable {
+			return testhelpers.NewQueryableWithSeries(testhelpers.FixtureMultipleSeries())
+		}),
+	})
+
+	t.Run("GET /api/v1/query returns vector with >= 1 sample", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/query", "query", "up").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "vector").
+			RequireLenAtLeast("$.data.result", 1)
+	})
+
+	t.Run("GET /api/v1/query_range returns matrix result type", func(t *testing.T) {
+		// Use relative timestamps to match our fixtures.
+		now := time.Now().Unix()
+		testhelpers.GET(t, api, "/api/v1/query_range",
+			"query", "up",
+			"start", strconv.FormatInt(now-120, 10),
+			"end", strconv.FormatInt(now, 10),
+			"step", "60").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "matrix")
+		// Note: Result may be empty if timestamps don't align perfectly with samples.
+	})
+
+	t.Run("GET /api/v1/labels returns non-empty array", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/labels").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONArray("$.data").
+			RequireLenAtLeast("$.data", 1)
+	})
+
+	t.Run("GET /api/v1/label/__name__/values contains expected metric names", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/label/__name__/values").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireArrayContains("$.data", "up").
+			RequireArrayContains("$.data", "http_requests_total")
+	})
+
+	t.Run("GET /api/v1/label/job/values contains expected jobs", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/label/job/values").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONArray("$.data").
+			RequireArrayContains("$.data", "prometheus").
+			RequireArrayContains("$.data", "node").
+			RequireArrayContains("$.data", "api")
+	})
+
+	t.Run("GET /api/v1/series with match returns results", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/series",
+			"match[]", "up",
+			"start", "0",
+			"end", "120").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONArray("$.data").
+			RequireLenAtLeast("$.data", 1)
+	})
+
+	t.Run("GET /api/v1/query with specific job returns filtered results", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/query", "query", `up{job="prometheus"}`).
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "vector").
+			RequireLenAtLeast("$.data.result", 1)
+	})
+
+	t.Run("GET /api/v1/query with aggregation returns result", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/query", "query", "sum(up)").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "vector")
+	})
+
+	t.Run("POST /api/v1/query returns vector with data", func(t *testing.T) {
+		testhelpers.POST(t, api, "/api/v1/query", "query", "up").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "vector").
+			RequireLenAtLeast("$.data.result", 1)
+	})
+}
+
+// TestAPIWithRules tests the API with rules configured.
+func TestAPIWithRules(t *testing.T) {
+	// Create an API with rule groups.
+	api := newTestAPI(t, testhelpers.APIConfig{
+		RulesRetriever: testhelpers.NewLazyLoader(func() testhelpers.RulesRetriever {
+			return testhelpers.NewRulesRetrieverWithGroups(testhelpers.FixtureRuleGroups())
+		}),
+	})
+
+	t.Run("GET /api/v1/rules returns groups with rules", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/rules").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data.groups").
+			RequireLenAtLeast("$.data.groups", 1).
+			RequireSome("$.data.groups", func(group any) bool {
+				if g, ok := group.(map[string]any); ok {
+					return g["name"] == "example"
+				}
+				return false
+			}).
+			RequireSome("$.data.groups", func(group any) bool {
+				if g, ok := group.(map[string]any); ok {
+					if g["name"] == "example" {
+						// Check that the group has rules.
+						if rules, ok := g["rules"].([]any); ok {
+							return len(rules) > 0
+						}
+					}
+				}
+				return false
+			})
+	})
+
+	t.Run("GET /api/v1/alerts returns alerts array", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/alerts").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data.alerts").
+			RequireJSONArray("$.data.alerts")
+	})
+
+	t.Run("GET /api/v1/rules with rule_name filter", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/rules", "rule_name[]", "InstanceDown").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONPathExists("$.data.groups")
+	})
+}
+
+// TestAPITSDBNotReady tests the API when TSDB is not ready (e.g., during WAL replay).
+// TSDB not ready errors are converted to errorUnavailable by setUnavailStatusOnTSDBNotReady,
+// which returns HTTP 500 Internal Server Error (the default for errorUnavailable).
+func TestAPITSDBNotReady(t *testing.T) {
+	// Create an API with a queryable that returns tsdb.ErrNotReady.
+	api := newTestAPI(t, testhelpers.APIConfig{
+		Queryable: testhelpers.NewLazyLoader(testhelpers.NewTSDBNotReadyQueryable),
+	})
+
+	t.Run("GET /api/v1/query returns 500 when TSDB not ready", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/query", "query", "up").
+			RequireStatusCode(500).
+			ValidateOpenAPI().
+			RequireError()
+	})
+
+	t.Run("POST /api/v1/query returns 500 when TSDB not ready", func(t *testing.T) {
+		testhelpers.POST(t, api, "/api/v1/query", "query", "up").
+			RequireStatusCode(500).
+			ValidateOpenAPI().
+			RequireError()
+	})
+
+	t.Run("GET /api/v1/query_range returns 500 when TSDB not ready", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/query_range",
+			"query", "up",
+			"start", "0",
+			"end", "100",
+			"step", "10").
+			RequireStatusCode(500).
+			ValidateOpenAPI().
+			RequireError()
+	})
+
+	t.Run("GET /api/v1/series returns 500 when TSDB not ready", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/series",
+			"match[]", "up",
+			"start", "0",
+			"end", "100").
+			RequireStatusCode(500).
+			ValidateOpenAPI().
+			RequireError()
+	})
+
+	t.Run("GET /api/v1/labels returns 500 when TSDB not ready", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/labels").
+			RequireStatusCode(500).
+			ValidateOpenAPI().
+			RequireError()
+	})
+
+	t.Run("GET /api/v1/label/{name}/values returns 500 when TSDB not ready", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/label/__name__/values").
+			RequireStatusCode(500).
+			ValidateOpenAPI().
+			RequireError()
+	})
+}
+
+// TestAPIWithNativeHistograms tests the API with native histogram data.
+func TestAPIWithNativeHistograms(t *testing.T) {
+	// Create an API with histogram series data.
+	api := newTestAPI(t, testhelpers.APIConfig{
+		Queryable: testhelpers.NewLazyLoader(func() storage.SampleAndChunkQueryable {
+			return testhelpers.NewQueryableWithSeries(testhelpers.FixtureHistogramSeries())
+		}),
+	})
+
+	t.Run("GET /api/v1/query returns vector with native histogram", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/query", "query", "test_histogram").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "vector").
+			RequireLenAtLeast("$.data.result", 1).
+			RequireSome("$.data.result", func(item any) bool {
+				sample, ok := item.(map[string]any)
+				if !ok {
+					return false
+				}
+				// Check that the sample has a histogram field (not a value field).
+				_, hasHistogram := sample["histogram"]
+				return hasHistogram
+			})
+	})
+
+	t.Run("POST /api/v1/query returns vector with native histogram", func(t *testing.T) {
+		testhelpers.POST(t, api, "/api/v1/query", "query", "test_histogram").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "vector").
+			RequireLenAtLeast("$.data.result", 1).
+			RequireSome("$.data.result", func(item any) bool {
+				sample, ok := item.(map[string]any)
+				if !ok {
+					return false
+				}
+				// Check that the sample has a histogram field (not a value field).
+				_, hasHistogram := sample["histogram"]
+				return hasHistogram
+			})
+	})
+
+	t.Run("GET /api/v1/query_range returns matrix with native histogram", func(t *testing.T) {
+		// Use relative timestamps to match our fixtures.
+		now := time.Now().Unix()
+		testhelpers.GET(t, api, "/api/v1/query_range",
+			"query", "test_histogram",
+			"start", strconv.FormatInt(now-120, 10),
+			"end", strconv.FormatInt(now, 10),
+			"step", "60").
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "matrix")
+	})
+
+	t.Run("GET /api/v1/query with histogram selector", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/query", "query", `test_histogram{job="prometheus"}`).
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireEquals("$.data.resultType", "vector").
+			RequireLenAtLeast("$.data.result", 1)
+	})
+
+	t.Run("GET /api/v1/series returns histogram metric series", func(t *testing.T) {
+		testhelpers.GET(t, api, "/api/v1/series",
+			"match[]", "test_histogram",
+			"start", "0",
+			"end", strconv.FormatInt(time.Now().Unix(), 10)).
+			RequireSuccess().
+			ValidateOpenAPI().
+			RequireJSONArray("$.data").
+			RequireLenAtLeast("$.data", 1)
+	})
+}

web/api/v1/api_test.go

@@ -166,8 +166,8 @@ func (t testTargetRetriever) TargetsDroppedCounts() map[string]int {
 	return r
 }
 
-func (testTargetRetriever) ScrapePoolConfig(_ string) (*config.ScrapeConfig, error) {
-	return &config.ScrapeConfig{
+func (testTargetRetriever) ScrapePoolConfig(pool string) (*config.ScrapeConfig, error) {
+	cfg := &config.ScrapeConfig{
 		RelabelConfigs: []*relabel.Config{
 			{
 				Action:               relabel.Replace,
@@ -182,20 +182,26 @@ func (testTargetRetriever) ScrapePoolConfig(_ string) (*config.ScrapeConfig, err
 				Regex:        relabel.MustNewRegexp(`example\.com:.*`),
 			},
 		},
-	}, nil
+	}
+	if pool == "testpool3" {
+		cfg.RelabelConfigs = append(cfg.RelabelConfigs, &relabel.Config{
+			Action:      relabel.Replace,
+			TargetLabel: "job",
+			Regex:       relabel.MustNewRegexp(".*"),
+			Replacement: "should_not_apply",
+		})
+	}
+	return cfg, nil
 }
 
 func (t *testTargetRetriever) SetMetadataStoreForTargets(identifier string, metadata scrape.MetricMetadataStore) error {
 	targets, ok := t.activeTargets[identifier]
-
 	if !ok {
-		return errors.New("targets not found")
+		return fmt.Errorf("no active target for %v", identifier)
 	}
-
 	for _, at := range targets {
 		at.SetMetadataStore(metadata)
 	}
-
 	return nil
 }
 
@@ -323,8 +329,8 @@ func (m *rulesRetrieverMock) CreateAlertingRules() {
 func (m *rulesRetrieverMock) CreateRuleGroups() {
 	m.CreateAlertingRules()
 	arules := m.AlertingRules()
-	storage := teststorage.New(m.testing)
-	defer storage.Close()
+	// Create separate storage for recordings to not pollute the main one.
+	s := teststorage.New(m.testing)
 
 	engineOpts := promql.EngineOpts{
 		Logger:     nil,
@@ -334,8 +340,8 @@ func (m *rulesRetrieverMock) CreateRuleGroups() {
 	}
 	engine := promqltest.NewTestEngineWithOpts(m.testing, engineOpts)
 	opts := &rules.ManagerOptions{
-		QueryFunc:  rules.EngineQueryFunc(engine, storage),
-		Appendable: storage,
+		QueryFunc:  rules.EngineQueryFunc(engine, s),
+		Appendable: s,
 		Context:    context.Background(),
 		Logger:     promslog.NewNopLogger(),
 		NotifyFunc: func(context.Context, string, ...*rules.Alert) {},
@@ -400,8 +406,23 @@ var sampleFlagMap = map[string]string{
 	"flag2": "value2",
 }
 
+func appendExemplars(t testing.TB, s storage.Storage, ex []exemplar.QueryResult) {
+	t.Helper()
+
+	// TODO(bwplotka): Use AppenderV2.AppendExemplar per series flow
+	// once its implemented: https://github.com/prometheus/prometheus/issues/17632#issuecomment-3759315095
+	app := s.Appender(t.Context())
+	for _, ed := range ex {
+		for _, e := range ed.Exemplars {
+			_, err := app.AppendExemplar(0, ed.SeriesLabels, e)
+			require.NoError(t, err)
+		}
+	}
+	require.NoError(t, app.Commit())
+}
+
 func TestEndpoints(t *testing.T) {
-	storage := promqltest.LoadedStorage(t, `
+	s := promqltest.LoadedStorage(t, `
 		load 1m
 			test_metric1{foo="bar"} 0+100x100
 			test_metric1{foo="boo"} 1+0x100
@@ -414,8 +435,8 @@ func TestEndpoints(t *testing.T) {
 			test_metric5{"host.name"="localhost"} 1+0x100
 			test_metric5{"junk\n{},=:  chars"="bar"} 1+0x100
 	`)
-	t.Cleanup(func() { storage.Close() })
 
+	// Add exemplar testdata here, given promqltest does not support exemplars.
 	start := time.Unix(0, 0)
 	exemplars := []exemplar.QueryResult{
 		{
@@ -459,15 +480,10 @@ func TestEndpoints(t *testing.T) {
 			},
 		},
 	}
-	for _, ed := range exemplars {
-		_, err := storage.AppendExemplar(0, ed.SeriesLabels, ed.Exemplars[0])
-		require.NoError(t, err, "failed to add exemplar: %+v", ed.Exemplars[0])
-	}
+	appendExemplars(t, s, exemplars)
 
 	now := time.Now()
-
 	ng := testEngine(t)
-
 	t.Run("local", func(t *testing.T) {
 		algr := rulesRetrieverMock{testing: t}
 
@@ -480,9 +496,9 @@ func TestEndpoints(t *testing.T) {
 		testTargetRetriever := setupTestTargetRetriever(t)
 
 		api := &API{
-			Queryable:             storage,
+			Queryable:             s,
 			QueryEngine:           ng,
-			ExemplarQueryable:     storage.ExemplarQueryable(),
+			ExemplarQueryable:     s,
 			targetRetriever:       testTargetRetriever.toFactory(),
 			alertmanagerRetriever: testAlertmanagerRetriever{}.toFactory(),
 			flagsMap:              sampleFlagMap,
@@ -491,14 +507,14 @@ func TestEndpoints(t *testing.T) {
 			ready:                 func(f http.HandlerFunc) http.HandlerFunc { return f },
 			rulesRetriever:        algr.toFactory(),
 		}
-		testEndpoints(t, api, testTargetRetriever, storage, true)
+		testEndpoints(t, api, testTargetRetriever, true)
 	})
 
 	// Run all the API tests against an API that is wired to forward queries via
 	// the remote read client to a test server, which in turn sends them to the
 	// data from the test storage.
 	t.Run("remote", func(t *testing.T) {
-		server := setupRemote(storage)
+		server := setupRemote(s)
 		defer server.Close()
 
 		u, err := url.Parse(server.URL)
@@ -520,6 +536,7 @@ func TestEndpoints(t *testing.T) {
 		remote := remote.NewStorage(promslog.New(&promslogConfig), prometheus.DefaultRegisterer, func() (int64, error) {
 			return 0, nil
 		}, dbDir, 1*time.Second, nil, false)
+		t.Cleanup(func() { _ = remote.Close() })
 
 		err = remote.ApplyConfig(&config.Config{
 			RemoteReadConfigs: []*config.RemoteReadConfig{
@@ -545,7 +562,7 @@ func TestEndpoints(t *testing.T) {
 		api := &API{
 			Queryable:             remote,
 			QueryEngine:           ng,
-			ExemplarQueryable:     storage.ExemplarQueryable(),
+			ExemplarQueryable:     s,
 			targetRetriever:       testTargetRetriever.toFactory(),
 			alertmanagerRetriever: testAlertmanagerRetriever{}.toFactory(),
 			flagsMap:              sampleFlagMap,
@@ -554,7 +571,7 @@ func TestEndpoints(t *testing.T) {
 			ready:                 func(f http.HandlerFunc) http.HandlerFunc { return f },
 			rulesRetriever:        algr.toFactory(),
 		}
-		testEndpoints(t, api, testTargetRetriever, storage, false)
+		testEndpoints(t, api, testTargetRetriever, false)
 	})
 }
 
@@ -567,7 +584,7 @@ func (b byLabels) Less(i, j int) bool { return labels.Compare(b[i], b[j]) < 0 }
 func TestGetSeries(t *testing.T) {
 	// TestEndpoints doesn't have enough label names to test api.labelNames
 	// endpoint properly. Hence we test it separately.
-	storage := promqltest.LoadedStorage(t, `
+	s := promqltest.LoadedStorage(t, `
 		load 1m
 			test_metric1{foo1="bar", baz="abc"} 0+100x100
 			test_metric1{foo2="boo"} 1+0x100
@@ -575,9 +592,9 @@ func TestGetSeries(t *testing.T) {
 			test_metric2{foo="boo", xyz="qwerty"} 1+0x100
 			test_metric2{foo="baz", abc="qwerty"} 1+0x100
 	`)
-	t.Cleanup(func() { storage.Close() })
+
 	api := &API{
-		Queryable: storage,
+		Queryable: s,
 	}
 	request := func(method string, matchers ...string) (*http.Request, error) {
 		u, err := url.Parse("http://example.com")
@@ -671,7 +688,7 @@ func TestGetSeries(t *testing.T) {
 
 func TestQueryExemplars(t *testing.T) {
 	start := time.Unix(0, 0)
-	storage := promqltest.LoadedStorage(t, `
+	s := promqltest.LoadedStorage(t, `
 		load 1m
 			test_metric1{foo="bar"} 0+100x100
 			test_metric1{foo="boo"} 1+0x100
@@ -682,12 +699,11 @@ func TestQueryExemplars(t *testing.T) {
 			test_metric4{foo="boo", dup="1"} 1+0x100
 			test_metric4{foo="boo"} 1+0x100
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	api := &API{
-		Queryable:         storage,
+		Queryable:         s,
 		QueryEngine:       testEngine(t),
-		ExemplarQueryable: storage.ExemplarQueryable(),
+		ExemplarQueryable: s,
 	}
 
 	request := func(method string, qs url.Values) (*http.Request, error) {
@@ -765,15 +781,10 @@ func TestQueryExemplars(t *testing.T) {
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			es := storage
+			es := s
 			ctx := context.Background()
 
-			for _, te := range tc.exemplars {
-				for _, e := range te.Exemplars {
-					_, err := es.AppendExemplar(0, te.SeriesLabels, e)
-					require.NoError(t, err)
-				}
-			}
+			appendExemplars(t, es, tc.exemplars)
 
 			req, err := request(http.MethodGet, tc.query)
 			require.NoError(t, err)
@@ -790,7 +801,7 @@ func TestQueryExemplars(t *testing.T) {
 func TestLabelNames(t *testing.T) {
 	// TestEndpoints doesn't have enough label names to test api.labelNames
 	// endpoint properly. Hence we test it separately.
-	storage := promqltest.LoadedStorage(t, `
+	s := promqltest.LoadedStorage(t, `
 		load 1m
 			test_metric1{foo1="bar", baz="abc"} 0+100x100
 			test_metric1{foo2="boo"} 1+0x100
@@ -798,9 +809,9 @@ func TestLabelNames(t *testing.T) {
 			test_metric2{foo="boo", xyz="qwerty"} 1+0x100
 			test_metric2{foo="baz", abc="qwerty"} 1+0x100
 	`)
-	t.Cleanup(func() { storage.Close() })
+
 	api := &API{
-		Queryable: storage,
+		Queryable: s,
 	}
 	request := func(method, limit string, matchers ...string) (*http.Request, error) {
 		u, err := url.Parse("http://example.com")
@@ -900,11 +911,10 @@ func (testStats) Builtin() (_ stats.BuiltinStats) {
 }
 
 func TestStats(t *testing.T) {
-	storage := teststorage.New(t)
-	t.Cleanup(func() { storage.Close() })
+	s := teststorage.New(t)
 
 	api := &API{
-		Queryable:   storage,
+		Queryable:   s,
 		QueryEngine: testEngine(t),
 		now: func() time.Time {
 			return time.Unix(123, 0)
@@ -1119,7 +1129,7 @@ func setupRemote(s storage.Storage) *httptest.Server {
 	return httptest.NewServer(handler)
 }
 
-func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.ExemplarStorage, testLabelAPI bool) {
+func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, testLabelAPI bool) {
 	start := time.Unix(0, 0)
 
 	type targetMetadata struct {
@@ -1139,7 +1149,6 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 		errType               errorType
 		sorter                func(any)
 		metadata              []targetMetadata
-		exemplars             []exemplar.QueryResult
 		zeroFunc              func(any)
 	}
 
@@ -1937,6 +1946,47 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 				},
 			},
 		},
+		{
+			endpoint: api.targetRelabelSteps,
+			query:    url.Values{"scrapePool": []string{"testpool3"}, "labels": []string{`{"job":"test","__address__":"localhost:9090"}`}},
+			response: &RelabelStepsResponse{
+				Steps: []RelabelStep{
+					{
+						Rule: &relabel.Config{
+							Action:               relabel.Replace,
+							Replacement:          "example.com:443",
+							TargetLabel:          "__address__",
+							Regex:                relabel.MustNewRegexp(""),
+							NameValidationScheme: model.LegacyValidation,
+						},
+						Output: labels.FromMap(map[string]string{
+							"job":         "test",
+							"__address__": "example.com:443",
+						}),
+						Keep: true,
+					},
+					{
+						Rule: &relabel.Config{
+							Action:       relabel.Drop,
+							SourceLabels: []model.LabelName{"__address__"},
+							Regex:        relabel.MustNewRegexp(`example\.com:.*`),
+						},
+						Output: labels.EmptyLabels(),
+						Keep:   false,
+					},
+					{
+						Rule: &relabel.Config{
+							Action:      relabel.Replace,
+							TargetLabel: "job",
+							Regex:       relabel.MustNewRegexp(".*"),
+							Replacement: "should_not_apply",
+						},
+						Output: labels.EmptyLabels(),
+						Keep:   false,
+					},
+				},
+			},
+		},
 		// With a matching metric.
 		{
 			endpoint: api.targetMetadata,
@@ -2047,8 +2097,8 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 			},
 			sorter: func(m any) {
 				sort.Slice(m.([]metricMetadata), func(i, j int) bool {
-					s := m.([]metricMetadata)
-					return s[i].MetricFamily < s[j].MetricFamily
+					mm := m.([]metricMetadata)
+					return mm[i].MetricFamily < mm[j].MetricFamily
 				})
 			},
 		},
@@ -3762,17 +3812,16 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
 
 					tr.ResetMetadataStore()
 					for _, tm := range test.metadata {
-						tr.SetMetadataStoreForTargets(tm.identifier, &testMetaStore{Metadata: tm.metadata})
-					}
-
-					for _, te := range test.exemplars {
-						for _, e := range te.Exemplars {
-							_, err := es.AppendExemplar(0, te.SeriesLabels, e)
-							require.NoError(t, err)
-						}
+						// TODO: Check error and fixed broken test/bug.
+						// TestEndpoints/local/run_60_metricMetadata_"limit=1&limit_per_metric=1"/GET fails if we check the error.
+						_ = tr.SetMetadataStoreForTargets(tm.identifier, &testMetaStore{Metadata: tm.metadata})
 					}
 
 					res := test.endpoint(req.WithContext(ctx))
+					if res.finalizer != nil {
+						// Finalizers were added to ensure closed readers on API panics, ensure they are closed here too.
+						res.finalizer()
+					}
 					assertAPIError(t, res.err, test.errType)
 
 					if test.sorter != nil {
@@ -4770,13 +4819,10 @@ func TestExtractQueryOpts(t *testing.T) {
 
 // Test query timeout parameter.
 func TestQueryTimeout(t *testing.T) {
-	storage := promqltest.LoadedStorage(t, `
+	s := promqltest.LoadedStorage(t, `
 		load 1m
 			test_metric1{foo="bar"} 0+100x100
 	`)
-	t.Cleanup(func() {
-		_ = storage.Close()
-	})
 
 	now := time.Now()
 
@@ -4796,9 +4842,9 @@ func TestQueryTimeout(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			engine := &fakeEngine{}
 			api := &API{
-				Queryable:             storage,
+				Queryable:             s,
 				QueryEngine:           engine,
-				ExemplarQueryable:     storage.ExemplarQueryable(),
+				ExemplarQueryable:     s,
 				alertmanagerRetriever: testAlertmanagerRetriever{}.toFactory(),
 				flagsMap:              sampleFlagMap,
 				now:                   func() time.Time { return now },

web/api/v1/errors_test.go

@@ -169,6 +169,7 @@ func createPrometheusAPI(t *testing.T, q storage.SampleAndChunkQueryable, overri
 		false,
 		overrideErrorCode,
 		nil,
+		OpenAPIOptions{},
 	)
 
 	promRouter := route.New().WithPrefix("/api/v1")

web/api/v1/openapi.go

@@ -0,0 +1,320 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file implements OpenAPI 3.2 specification generation for the Prometheus HTTP API.
+// It provides dynamic spec building with optional path filtering.
+package v1
+
+import (
+	"log/slog"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+	"sync"
+
+	"github.com/pb33f/libopenapi/datamodel/high/base"
+	v3 "github.com/pb33f/libopenapi/datamodel/high/v3"
+	"github.com/pb33f/libopenapi/orderedmap"
+)
+
+const (
+	// OpenAPI 3.1.0 is the default version with broader compatibility.
+	openAPIVersion31 = "3.1.0"
+	// OpenAPI 3.2.0 supports advanced features like itemSchema for SSE streams.
+	openAPIVersion32 = "3.2.0"
+)
+
+// OpenAPIOptions configures the OpenAPI spec builder.
+type OpenAPIOptions struct {
+	// IncludePaths filters which paths to include in the spec.
+	// If empty, all paths are included.
+	// Paths are matched by prefix (e.g., "/query" matches "/query" and "/query_range").
+	IncludePaths []string
+
+	// ExternalURL is the external URL of the Prometheus server (e.g., "http://prometheus.example.com:9090").
+	ExternalURL string
+
+	// Version is the API version to include in the OpenAPI spec.
+	// If empty, defaults to "0.0.1-undefined".
+	Version string
+}
+
+// OpenAPIBuilder builds and caches OpenAPI specifications.
+type OpenAPIBuilder struct {
+	mu           sync.RWMutex
+	cachedYAML31 []byte // Cached OpenAPI 3.1 spec.
+	cachedYAML32 []byte // Cached OpenAPI 3.2 spec.
+	options      OpenAPIOptions
+	logger       *slog.Logger
+}
+
+// NewOpenAPIBuilder creates a new OpenAPI builder with the given options.
+func NewOpenAPIBuilder(opts OpenAPIOptions, logger *slog.Logger) *OpenAPIBuilder {
+	b := &OpenAPIBuilder{
+		options: opts,
+		logger:  logger,
+	}
+
+	b.rebuild()
+	return b
+}
+
+// rebuild constructs the OpenAPI specs for both 3.1 and 3.2 versions based on current options.
+func (b *OpenAPIBuilder) rebuild() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// Build OpenAPI 3.1 spec.
+	doc31 := b.buildDocument(openAPIVersion31)
+	yamlBytes31, err := doc31.Render()
+	if err != nil {
+		b.logger.Error("failed to render OpenAPI 3.1 spec - this is a bug, please report it", "err", err)
+		return
+	}
+	b.cachedYAML31 = yamlBytes31
+
+	// Build OpenAPI 3.2 spec.
+	doc32 := b.buildDocument(openAPIVersion32)
+	yamlBytes32, err := doc32.Render()
+	if err != nil {
+		b.logger.Error("failed to render OpenAPI 3.2 spec - this is a bug, please report it", "err", err)
+		return
+	}
+	b.cachedYAML32 = yamlBytes32
+}
+
+// ServeOpenAPI returns the OpenAPI specification as YAML.
+// By default, serves OpenAPI 3.1.0. Use ?openapi_version=3.2 for OpenAPI 3.2.0.
+func (b *OpenAPIBuilder) ServeOpenAPI(w http.ResponseWriter, r *http.Request) {
+	// Parse query parameter to determine which version to serve.
+	requestedVersion := r.URL.Query().Get("openapi_version")
+
+	b.mu.RLock()
+	var yamlData []byte
+	switch requestedVersion {
+	case "3.2", "3.2.0":
+		yamlData = b.cachedYAML32
+	case "3.1", "3.1.0":
+		yamlData = b.cachedYAML31
+	default:
+		// Default to OpenAPI 3.1.0 for broader compatibility.
+		yamlData = b.cachedYAML31
+	}
+	b.mu.RUnlock()
+
+	w.Header().Set("Content-Type", "application/yaml; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+	w.WriteHeader(http.StatusOK)
+	w.Write(yamlData)
+}
+
+// WrapHandler returns the handler unchanged (no validation).
+func (*OpenAPIBuilder) WrapHandler(next http.HandlerFunc) http.HandlerFunc {
+	return next
+}
+
+// shouldIncludePath checks if a path should be included based on options.
+func (b *OpenAPIBuilder) shouldIncludePath(path string) bool {
+	if len(b.options.IncludePaths) == 0 {
+		return true
+	}
+	for _, include := range b.options.IncludePaths {
+		if strings.HasPrefix(path, include) || path == include {
+			return true
+		}
+	}
+	return false
+}
+
+// shouldIncludePathForVersion checks if a path should be included for a specific OpenAPI version.
+func (b *OpenAPIBuilder) shouldIncludePathForVersion(path, version string) bool {
+	// First check IncludePaths filter.
+	if !b.shouldIncludePath(path) {
+		return false
+	}
+
+	// OpenAPI 3.1 excludes paths that require 3.2 features.
+	// The /notifications/live endpoint uses itemSchema which is a 3.2-only feature.
+	if version == openAPIVersion31 && path == "/notifications/live" {
+		return false
+	}
+
+	return true
+}
+
+// buildDocument creates the OpenAPI document for the specified version using high-level structs.
+func (b *OpenAPIBuilder) buildDocument(version string) *v3.Document {
+	return &v3.Document{
+		Version:    version,
+		Info:       b.buildInfo(),
+		Servers:    b.buildServers(),
+		Tags:       b.buildTags(version),
+		Paths:      b.buildPaths(version),
+		Components: b.buildComponents(),
+	}
+}
+
+// buildInfo constructs the info section.
+func (b *OpenAPIBuilder) buildInfo() *base.Info {
+	apiVersion := b.options.Version
+	if apiVersion == "" {
+		apiVersion = "0.0.1-undefined"
+	}
+	return &base.Info{
+		Title:       "Prometheus API",
+		Description: "Prometheus is an Open-Source monitoring system with a dimensional data model, flexible query language, efficient time series database and modern alerting approach.",
+		Version:     apiVersion,
+		Contact: &base.Contact{
+			Name: "Prometheus Community",
+			URL:  "https://prometheus.io/community/",
+		},
+	}
+}
+
+// buildServers constructs the servers section.
+func (b *OpenAPIBuilder) buildServers() []*v3.Server {
+	// ExternalURL is always set by computeExternalURL in main.go.
+	// It includes scheme, host, port, and optional path prefix (without trailing slash).
+	serverURL := "/api/v1"
+	if b.options.ExternalURL != "" {
+		baseURL, err := url.Parse(b.options.ExternalURL)
+		if err == nil {
+			// Use path.Join to properly append /api/v1 to the existing path.
+			// Then use ResolveReference to construct the full URL.
+			baseURL.Path = path.Join(baseURL.Path, "/api/v1")
+			serverURL = baseURL.String()
+		}
+	}
+	return []*v3.Server{
+		{URL: serverURL},
+	}
+}
+
+// buildTags constructs the global tags list.
+// Tag summary is an OpenAPI 3.2 feature, excluded from 3.1.
+// Tag description is supported in both 3.1 and 3.2.
+func (*OpenAPIBuilder) buildTags(version string) []*base.Tag {
+	// Define tags with all metadata.
+	tagData := []struct {
+		name        string
+		summary     string
+		description string
+	}{
+		{"query", "Query", "Query and evaluate PromQL expressions."},
+		{"metadata", "Metadata", "Retrieve metric metadata such as type and unit."},
+		{"labels", "Labels", "Query label names and values."},
+		{"series", "Series", "Query and manage time series."},
+		{"targets", "Targets", "Retrieve target and scrape pool information."},
+		{"rules", "Rules", "Query recording and alerting rules."},
+		{"alerts", "Alerts", "Query active alerts and alertmanager discovery."},
+		{"status", "Status", "Retrieve server status and configuration."},
+		{"admin", "Admin", "Administrative operations for TSDB management."},
+		{"features", "Features", "Query enabled features."},
+		{"remote", "Remote Storage", "Remote read and write endpoints."},
+		{"otlp", "OTLP", "OpenTelemetry Protocol metrics ingestion."},
+		{"notifications", "Notifications", "Server notifications and events."},
+	}
+
+	tags := make([]*base.Tag, 0, len(tagData))
+	for _, td := range tagData {
+		tag := &base.Tag{
+			Name:        td.name,
+			Description: td.description, // Description is supported in both 3.1 and 3.2.
+		}
+
+		// Summary is an OpenAPI 3.2 feature only.
+		if version == openAPIVersion32 {
+			tag.Summary = td.summary
+		}
+
+		tags = append(tags, tag)
+	}
+
+	return tags
+}
+
+// buildPaths constructs all API path definitions.
+func (b *OpenAPIBuilder) buildPaths(version string) *v3.Paths {
+	pathItems := orderedmap.New[string, *v3.PathItem]()
+
+	allPaths := b.getAllPathDefinitions()
+	for pair := allPaths.First(); pair != nil; pair = pair.Next() {
+		if b.shouldIncludePathForVersion(pair.Key(), version) {
+			pathItems.Set(pair.Key(), pair.Value())
+		}
+	}
+
+	return &v3.Paths{PathItems: pathItems}
+}
+
+// getAllPathDefinitions returns all path definitions.
+func (b *OpenAPIBuilder) getAllPathDefinitions() *orderedmap.Map[string, *v3.PathItem] {
+	paths := orderedmap.New[string, *v3.PathItem]()
+
+	// Query endpoints.
+	paths.Set("/query", b.queryPath())
+	paths.Set("/query_range", b.queryRangePath())
+	paths.Set("/query_exemplars", b.queryExemplarsPath())
+	paths.Set("/format_query", b.formatQueryPath())
+	paths.Set("/parse_query", b.parseQueryPath())
+
+	// Label endpoints.
+	paths.Set("/labels", b.labelsPath())
+	paths.Set("/label/{name}/values", b.labelValuesPath())
+
+	// Series endpoints.
+	paths.Set("/series", b.seriesPath())
+
+	// Metadata endpoints.
+	paths.Set("/metadata", b.metadataPath())
+
+	// Target endpoints.
+	paths.Set("/scrape_pools", b.scrapePoolsPath())
+	paths.Set("/targets", b.targetsPath())
+	paths.Set("/targets/metadata", b.targetsMetadataPath())
+	paths.Set("/targets/relabel_steps", b.targetsRelabelStepsPath())
+
+	// Rules and alerts endpoints.
+	paths.Set("/rules", b.rulesPath())
+	paths.Set("/alerts", b.alertsPath())
+	paths.Set("/alertmanagers", b.alertmanagersPath())
+
+	// Status endpoints.
+	paths.Set("/status/config", b.statusConfigPath())
+	paths.Set("/status/runtimeinfo", b.statusRuntimeInfoPath())
+	paths.Set("/status/buildinfo", b.statusBuildInfoPath())
+	paths.Set("/status/flags", b.statusFlagsPath())
+	paths.Set("/status/tsdb", b.statusTSDBPath())
+	paths.Set("/status/tsdb/blocks", b.statusTSDBBlocksPath())
+	paths.Set("/status/walreplay", b.statusWALReplayPath())
+
+	// Admin endpoints.
+	paths.Set("/admin/tsdb/delete_series", b.adminDeleteSeriesPath())
+	paths.Set("/admin/tsdb/clean_tombstones", b.adminCleanTombstonesPath())
+	paths.Set("/admin/tsdb/snapshot", b.adminSnapshotPath())
+
+	// Remote endpoints.
+	paths.Set("/read", b.remoteReadPath())
+	paths.Set("/write", b.remoteWritePath())
+	paths.Set("/otlp/v1/metrics", b.otlpWritePath())
+
+	// Notifications endpoints.
+	paths.Set("/notifications", b.notificationsPath())
+	paths.Set("/notifications/live", b.notificationsLivePath())
+
+	// Features endpoint.
+	paths.Set("/features", b.featuresPath())
+
+	return paths
+}

web/api/v1/openapi_coverage_test.go

@@ -0,0 +1,258 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+import (
+	_ "embed"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"strconv"
+	"strings"
+	"testing"
+
+	v3 "github.com/pb33f/libopenapi/datamodel/high/v3"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+)
+
+//go:embed api.go
+var apiGoSource string
+
+// routeInfo represents a route extracted from the Register function.
+type routeInfo struct {
+	method string
+	path   string
+}
+
+// extractRoutesFromRegister parses the api.go source and extracts all routes
+// registered in the (*API) Register function using AST.
+func extractRoutesFromRegister(t *testing.T, source string) []routeInfo {
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "api.go", source, parser.ParseComments)
+	require.NoError(t, err, "failed to parse api.go")
+
+	var registerFunc *ast.FuncDecl
+
+	// Find the Register method on *API.
+	ast.Inspect(f, func(n ast.Node) bool {
+		fn, ok := n.(*ast.FuncDecl)
+		if !ok || fn.Body == nil {
+			return true
+		}
+
+		if fn.Name.Name != "Register" {
+			return true
+		}
+
+		// Ensure it's a method on *API.
+		if fn.Recv == nil || len(fn.Recv.List) != 1 {
+			return true
+		}
+
+		star, ok := fn.Recv.List[0].Type.(*ast.StarExpr)
+		if !ok {
+			return true
+		}
+
+		ident, ok := star.X.(*ast.Ident)
+		if !ok || ident.Name != "API" {
+			return true
+		}
+
+		registerFunc = fn
+		return false // Stop walking once found.
+	})
+
+	require.NotNil(t, registerFunc, "Register method not found")
+
+	var routes []routeInfo
+
+	// Extract all r.Get, r.Post, r.Put, r.Delete, r.Options calls.
+	ast.Inspect(registerFunc.Body, func(n ast.Node) bool {
+		call, ok := n.(*ast.CallExpr)
+		if !ok {
+			return true
+		}
+
+		sel, ok := call.Fun.(*ast.SelectorExpr)
+		if !ok {
+			return true
+		}
+
+		// Check if it's a router method call.
+		method := sel.Sel.Name
+		if method != "Get" && method != "Post" && method != "Put" && method != "Delete" && method != "Del" && method != "Options" {
+			return true
+		}
+
+		// Ensure the receiver is 'r'.
+		if x, ok := sel.X.(*ast.Ident); !ok || x.Name != "r" {
+			return true
+		}
+
+		if len(call.Args) == 0 {
+			return true
+		}
+
+		// Extract the path from the first argument.
+		lit, ok := call.Args[0].(*ast.BasicLit)
+		if !ok || lit.Kind != token.STRING {
+			return true
+		}
+
+		path, err := strconv.Unquote(lit.Value)
+		if err != nil {
+			return true
+		}
+
+		// Normalize Del to DELETE.
+		if method == "Del" {
+			method = "Delete"
+		}
+
+		routes = append(routes, routeInfo{
+			method: strings.ToUpper(method),
+			path:   path,
+		})
+		return true
+	})
+
+	return routes
+}
+
+// normalizePathForOpenAPI converts route paths with colon parameters to OpenAPI format.
+// e.g., "/label/:name/values" -> "/label/{name}/values".
+func normalizePathForOpenAPI(path string) string {
+	// Replace :param with {param}.
+	parts := strings.Split(path, "/")
+	for i, part := range parts {
+		if trimmed, ok := strings.CutPrefix(part, ":"); ok {
+			parts[i] = "{" + trimmed + "}"
+		}
+	}
+	return strings.Join(parts, "/")
+}
+
+// TestOpenAPICoverage verifies that all routes registered in the Register function
+// are documented in the OpenAPI specification.
+func TestOpenAPICoverage(t *testing.T) {
+	// Extract routes from api.go using AST.
+	routes := extractRoutesFromRegister(t, apiGoSource)
+	require.NotEmpty(t, routes, "no routes found in Register function")
+
+	// Build OpenAPI spec.
+	builder := NewOpenAPIBuilder(OpenAPIOptions{}, promslog.NewNopLogger())
+	allPaths := builder.getAllPathDefinitions()
+
+	// Create a map of OpenAPI paths for quick lookup.
+	// Key is the normalized path, value is the PathItem.
+	openAPIPaths := make(map[string]bool)
+	for pair := allPaths.First(); pair != nil; pair = pair.Next() {
+		pathItem := pair.Value()
+		path := pair.Key()
+
+		// Track which methods are defined for this path.
+		if pathItem.Get != nil {
+			openAPIPaths[path+":GET"] = true
+		}
+		if pathItem.Post != nil {
+			openAPIPaths[path+":POST"] = true
+		}
+		if pathItem.Put != nil {
+			openAPIPaths[path+":PUT"] = true
+		}
+		if pathItem.Delete != nil {
+			openAPIPaths[path+":DELETE"] = true
+		}
+		if pathItem.Options != nil {
+			openAPIPaths[path+":OPTIONS"] = true
+		}
+	}
+
+	// Check coverage for each route.
+	var missingRoutes []string
+	ignoredRoutes := map[string]bool{
+		"/*path:OPTIONS":          true, // Wildcard OPTIONS handler.
+		"/openapi.yaml:GET":       true, // Self-referential endpoint.
+		"/notifications/live:GET": true, // SSE endpoint (version-specific).
+	}
+
+	for _, route := range routes {
+		normalizedPath := normalizePathForOpenAPI(route.path)
+		key := normalizedPath + ":" + route.method
+
+		// Skip ignored routes.
+		if ignoredRoutes[key] {
+			continue
+		}
+
+		if !openAPIPaths[key] {
+			missingRoutes = append(missingRoutes, key)
+		}
+	}
+
+	if len(missingRoutes) > 0 {
+		t.Errorf("The following routes are registered but not documented in OpenAPI spec:\n%s",
+			strings.Join(missingRoutes, "\n"))
+	}
+}
+
+// TestOpenAPIHasNoExtraRoutes verifies that the OpenAPI spec doesn't document
+// routes that aren't actually registered.
+func TestOpenAPIHasNoExtraRoutes(t *testing.T) {
+	// Extract routes from api.go using AST.
+	routes := extractRoutesFromRegister(t, apiGoSource)
+	require.NotEmpty(t, routes, "no routes found in Register function")
+
+	// Create a map of registered routes.
+	registeredRoutes := make(map[string]bool)
+	for _, route := range routes {
+		normalizedPath := normalizePathForOpenAPI(route.path)
+		key := normalizedPath + ":" + route.method
+		registeredRoutes[key] = true
+	}
+
+	// Build OpenAPI spec.
+	builder := NewOpenAPIBuilder(OpenAPIOptions{}, promslog.NewNopLogger())
+	allPaths := builder.getAllPathDefinitions()
+
+	// Check if any OpenAPI paths are not registered.
+	var extraRoutes []string
+
+	for pair := allPaths.First(); pair != nil; pair = pair.Next() {
+		pathItem := pair.Value()
+		path := pair.Key()
+
+		checkMethod := func(method string, op *v3.Operation) {
+			if op != nil {
+				key := path + ":" + method
+				if !registeredRoutes[key] {
+					extraRoutes = append(extraRoutes, key)
+				}
+			}
+		}
+
+		checkMethod("GET", pathItem.Get)
+		checkMethod("POST", pathItem.Post)
+		checkMethod("PUT", pathItem.Put)
+		checkMethod("DELETE", pathItem.Delete)
+		checkMethod("OPTIONS", pathItem.Options)
+	}
+
+	if len(extraRoutes) > 0 {
+		t.Errorf("The following routes are documented in OpenAPI but not registered:\n%s",
+			strings.Join(extraRoutes, "\n"))
+	}
+}

web/api/v1/openapi_golden_test.go

@@ -0,0 +1,176 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+
+	"github.com/prometheus/prometheus/web/api/testhelpers"
+)
+
+var updateOpenAPISpec = flag.Bool("update-openapi-spec", false, "update openapi golden files with the current specs")
+
+// TestOpenAPIGolden_3_1 verifies that the OpenAPI 3.1 spec matches the golden file.
+func TestOpenAPIGolden_3_1(t *testing.T) {
+	// Create an API instance to serve the OpenAPI spec.
+	api := newTestAPI(t, testhelpers.APIConfig{})
+
+	// Fetch the OpenAPI 3.1 spec from the API (default, no query param).
+	resp := testhelpers.GET(t, api, "/api/v1/openapi.yaml")
+	require.Equal(t, 200, resp.StatusCode, "expected HTTP 200 for OpenAPI spec endpoint")
+	require.NotEmpty(t, resp.Body, "OpenAPI spec should not be empty")
+
+	goldenPath := filepath.Join("testdata", "openapi_3.1_golden.yaml")
+
+	if *updateOpenAPISpec {
+		// Update mode: write the current spec to the golden file.
+		t.Logf("Updating golden file: %s", goldenPath)
+
+		// Ensure the testdata directory exists.
+		err := os.MkdirAll(filepath.Dir(goldenPath), 0o755)
+		require.NoError(t, err, "failed to create testdata directory")
+
+		// Write the golden file.
+		err = os.WriteFile(goldenPath, []byte(resp.Body), 0o644)
+		require.NoError(t, err, "failed to write golden file")
+
+		t.Logf("Golden file updated successfully")
+		return
+	}
+
+	// Comparison mode: verify the spec matches the golden file.
+	goldenData, err := os.ReadFile(goldenPath)
+	require.NoError(t, err, "failed to read golden file (run with -update-openapi-spec to generate it)")
+
+	require.Equal(t, string(goldenData), resp.Body,
+		"OpenAPI 3.1 spec does not match golden file. Run 'go test -update-openapi-spec' to update.")
+
+	// Verify version field is 3.1.0.
+	var spec map[string]any
+	err = yaml.Unmarshal([]byte(resp.Body), &spec)
+	require.NoError(t, err)
+	require.Equal(t, "3.1.0", spec["openapi"], "OpenAPI version should be 3.1.0")
+
+	// Verify /notifications/live is NOT present in 3.1 spec.
+	paths := spec["paths"].(map[string]any)
+	_, found := paths["/notifications/live"]
+	require.False(t, found, "/notifications/live should not be in OpenAPI 3.1 spec")
+}
+
+// TestOpenAPIGolden_3_2 verifies that the OpenAPI 3.2 spec matches the golden file.
+func TestOpenAPIGolden_3_2(t *testing.T) {
+	// Create an API instance to serve the OpenAPI spec.
+	api := newTestAPI(t, testhelpers.APIConfig{})
+
+	// Fetch the OpenAPI 3.2 spec from the API with query parameter.
+	resp := testhelpers.GET(t, api, "/api/v1/openapi.yaml?openapi_version=3.2")
+	require.Equal(t, 200, resp.StatusCode, "expected HTTP 200 for OpenAPI spec endpoint")
+	require.NotEmpty(t, resp.Body, "OpenAPI spec should not be empty")
+
+	goldenPath := filepath.Join("testdata", "openapi_3.2_golden.yaml")
+
+	if *updateOpenAPISpec {
+		// Update mode: write the current spec to the golden file.
+		t.Logf("Updating golden file: %s", goldenPath)
+
+		// Ensure the testdata directory exists.
+		err := os.MkdirAll(filepath.Dir(goldenPath), 0o755)
+		require.NoError(t, err, "failed to create testdata directory")
+
+		// Write the golden file.
+		err = os.WriteFile(goldenPath, []byte(resp.Body), 0o644)
+		require.NoError(t, err, "failed to write golden file")
+
+		t.Logf("Golden file updated successfully")
+		return
+	}
+
+	// Comparison mode: verify the spec matches the golden file.
+	goldenData, err := os.ReadFile(goldenPath)
+	require.NoError(t, err, "failed to read golden file (run with -update-openapi-spec to generate it)")
+
+	require.Equal(t, string(goldenData), resp.Body,
+		"OpenAPI 3.2 spec does not match golden file. Run 'go test -update-openapi-spec' to update.")
+
+	// Verify version field is 3.2.0.
+	var spec map[string]any
+	err = yaml.Unmarshal([]byte(resp.Body), &spec)
+	require.NoError(t, err)
+	require.Equal(t, "3.2.0", spec["openapi"], "OpenAPI version should be 3.2.0")
+
+	// Verify /notifications/live IS present in 3.2 spec.
+	paths := spec["paths"].(map[string]any)
+	_, found := paths["/notifications/live"]
+	require.True(t, found, "/notifications/live should be in OpenAPI 3.2 spec")
+}
+
+// TestOpenAPIVersionSelection verifies version query parameter handling.
+func TestOpenAPIVersionSelection(t *testing.T) {
+	api := newTestAPI(t, testhelpers.APIConfig{})
+
+	tests := []struct {
+		name            string
+		url             string
+		expectedVersion string
+		expectLivePath  bool
+	}{
+		{
+			name:            "default to 3.1.0",
+			url:             "/api/v1/openapi.yaml",
+			expectedVersion: "3.1.0",
+			expectLivePath:  false,
+		},
+		{
+			name:            "explicit 3.1",
+			url:             "/api/v1/openapi.yaml?openapi_version=3.1",
+			expectedVersion: "3.1.0",
+			expectLivePath:  false,
+		},
+		{
+			name:            "explicit 3.2",
+			url:             "/api/v1/openapi.yaml?openapi_version=3.2",
+			expectedVersion: "3.2.0",
+			expectLivePath:  true,
+		},
+		{
+			name:            "invalid version defaults to 3.1.0",
+			url:             "/api/v1/openapi.yaml?openapi_version=4.0",
+			expectedVersion: "3.1.0",
+			expectLivePath:  false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			resp := testhelpers.GET(t, api, tc.url)
+			require.Equal(t, 200, resp.StatusCode)
+
+			var spec map[string]any
+			err := yaml.Unmarshal([]byte(resp.Body), &spec)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expectedVersion, spec["openapi"])
+
+			paths := spec["paths"].(map[string]any)
+			_, found := paths["/notifications/live"]
+			require.Equal(t, tc.expectLivePath, found)
+		})
+	}
+}

web/api/v1/openapi_helpers.go

@@ -0,0 +1,343 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+import (
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/pb33f/libopenapi/datamodel/high/base"
+	v3 "github.com/pb33f/libopenapi/datamodel/high/v3"
+	"github.com/pb33f/libopenapi/orderedmap"
+	yaml "go.yaml.in/yaml/v4"
+
+	"github.com/prometheus/prometheus/promql"
+)
+
+// Helper functions for building common structures.
+
+// exampleTime is a reference time used for timestamp examples.
+var exampleTime = time.Date(2026, 1, 2, 13, 37, 0, 0, time.UTC)
+
+func boolPtr(b bool) *bool {
+	return &b
+}
+
+func int64Ptr(i int64) *int64 {
+	return &i
+}
+
+type example struct {
+	name  string
+	value any
+}
+
+// exampleMap creates an Examples map from the provided examples.
+func exampleMap(exs []example) *orderedmap.Map[string, *base.Example] {
+	examples := orderedmap.New[string, *base.Example]()
+	for _, ex := range exs {
+		examples.Set(ex.name, &base.Example{
+			Value: createYAMLNode(ex.value),
+		})
+	}
+	return examples
+}
+
+func schemaRef(ref string) *base.SchemaProxy {
+	return base.CreateSchemaProxyRef(ref)
+}
+
+func schemaFromType(t string) *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{Type: []string{t}})
+}
+
+func stringSchema() *base.SchemaProxy {
+	return schemaFromType("string")
+}
+
+func integerSchema() *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:   []string{"integer"},
+		Format: "int64",
+	})
+}
+
+func stringSchemaWithDescription(description string) *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"string"},
+		Description: description,
+	})
+}
+
+func stringSchemaWithDescriptionAndExample(description string, example any) *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"string"},
+		Description: description,
+		Example:     createYAMLNode(example),
+	})
+}
+
+func integerSchemaWithDescription(description string) *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"integer"},
+		Format:      "int64",
+		Description: description,
+	})
+}
+
+func integerSchemaWithDescriptionAndExample(description string, example any) *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"integer"},
+		Format:      "int64",
+		Description: description,
+		Example:     createYAMLNode(example),
+	})
+}
+
+func stringArraySchemaWithDescription(description string) *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"array"},
+		Items:       &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		Description: description,
+	})
+}
+
+func stringArraySchemaWithDescriptionAndExample(description string, example any) *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"array"},
+		Items:       &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		Description: description,
+		Example:     createYAMLNode(example),
+	})
+}
+
+func statusSchema() *base.SchemaProxy {
+	successNode := &yaml.Node{Kind: yaml.ScalarNode, Value: "success"}
+	errorNode := &yaml.Node{Kind: yaml.ScalarNode, Value: "error"}
+	exampleNode := &yaml.Node{Kind: yaml.ScalarNode, Value: "success"}
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"string"},
+		Enum:        []*yaml.Node{successNode, errorNode},
+		Description: "Response status.",
+		Example:     exampleNode,
+	})
+}
+
+func warningsSchema() *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"array"},
+		Items:       &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		Description: "Only set if there were warnings while executing the request. There will still be data in the data field.",
+	})
+}
+
+func infosSchema() *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"array"},
+		Items:       &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		Description: "Only set if there were info-level annotations while executing the request.",
+	})
+}
+
+func timestampSchema() *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		OneOf: []*base.SchemaProxy{
+			base.CreateSchemaProxy(&base.Schema{
+				Type:        []string{"string"},
+				Format:      "date-time",
+				Description: "RFC3339 timestamp.",
+			}),
+			base.CreateSchemaProxy(&base.Schema{
+				Type:        []string{"number"},
+				Format:      "unixtime",
+				Description: "Unix timestamp in seconds.",
+			}),
+		},
+		Description: "Timestamp in RFC3339 format or Unix timestamp in seconds.",
+	})
+}
+
+func stringSchemaWithConstValue(value string) *base.SchemaProxy {
+	node := &yaml.Node{Kind: yaml.ScalarNode, Value: value}
+	return base.CreateSchemaProxy(&base.Schema{
+		Type: []string{"string"},
+		Enum: []*yaml.Node{node},
+	})
+}
+
+func dateTimeSchemaWithDescription(description string) *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"string"},
+		Format:      "date-time",
+		Description: description,
+	})
+}
+
+func numberSchemaWithDescription(description string) *base.SchemaProxy {
+	return base.CreateSchemaProxy(&base.Schema{
+		Type:        []string{"number"},
+		Format:      "double",
+		Description: description,
+	})
+}
+
+func errorResponse() *v3.Response {
+	content := orderedmap.New[string, *v3.MediaType]()
+	content.Set("application/json", &v3.MediaType{
+		Schema: schemaRef("#/components/schemas/Error"),
+	})
+	return &v3.Response{
+		Description: "Error",
+		Content:     content,
+	}
+}
+
+func noContentResponse() *v3.Response {
+	return &v3.Response{Description: "No Content"}
+}
+
+func responsesNoContent() *v3.Responses {
+	codes := orderedmap.New[string, *v3.Response]()
+	codes.Set("204", noContentResponse())
+	codes.Set("default", errorResponse())
+	return &v3.Responses{Codes: codes}
+}
+
+func pathParam(name, description string, schema *base.SchemaProxy) *v3.Parameter {
+	return &v3.Parameter{
+		Name:        name,
+		In:          "path",
+		Description: description,
+		Required:    boolPtr(true),
+		Schema:      schema,
+	}
+}
+
+// createYAMLNode converts Go data to yaml.Node for use in examples.
+func createYAMLNode(data any) *yaml.Node {
+	node := &yaml.Node{}
+	bytes, _ := yaml.Marshal(data)
+	_ = yaml.Unmarshal(bytes, node)
+	return node
+}
+
+// formRequestBodyWithExamples creates a form-encoded request body with examples.
+func formRequestBodyWithExamples(schemaRef string, examples *orderedmap.Map[string, *base.Example], description string) *v3.RequestBody {
+	content := orderedmap.New[string, *v3.MediaType]()
+	mediaType := &v3.MediaType{
+		Schema: base.CreateSchemaProxyRef("#/components/schemas/" + schemaRef),
+	}
+	if examples != nil {
+		mediaType.Examples = examples
+	}
+	content.Set("application/x-www-form-urlencoded", mediaType)
+	return &v3.RequestBody{
+		Required:    boolPtr(true),
+		Description: description,
+		Content:     content,
+	}
+}
+
+// jsonResponseWithExamples creates a JSON response with examples.
+func jsonResponseWithExamples(schemaRef string, examples *orderedmap.Map[string, *base.Example], description string) *v3.Response {
+	content := orderedmap.New[string, *v3.MediaType]()
+	mediaType := &v3.MediaType{
+		Schema: base.CreateSchemaProxyRef("#/components/schemas/" + schemaRef),
+	}
+	if examples != nil {
+		mediaType.Examples = examples
+	}
+	content.Set("application/json", mediaType)
+	return &v3.Response{
+		Description: description,
+		Content:     content,
+	}
+}
+
+// responsesWithErrorExamples creates responses with both success and error examples.
+func responsesWithErrorExamples(okSchemaRef string, successExamples, errorExamples *orderedmap.Map[string, *base.Example], successDescription, errorDescription string) *v3.Responses {
+	codes := orderedmap.New[string, *v3.Response]()
+	codes.Set("200", jsonResponseWithExamples(okSchemaRef, successExamples, successDescription))
+	codes.Set("default", jsonResponseWithExamples("Error", errorExamples, errorDescription))
+	return &v3.Responses{Codes: codes}
+}
+
+// timestampExamples returns examples for timestamp parameters (RFC3339 and epoch).
+func timestampExamples(t time.Time) []example {
+	return []example{
+		{"RFC3339", t.Format(time.RFC3339Nano)},
+		{"epoch", t.Unix()},
+	}
+}
+
+// queryParamWithExample creates a query parameter with examples.
+func queryParamWithExample(name, description string, required bool, schema *base.SchemaProxy, examples []example) *v3.Parameter {
+	param := &v3.Parameter{
+		Name:        name,
+		In:          "query",
+		Description: description,
+		Required:    &required,
+		Explode:     boolPtr(false),
+		Schema:      schema,
+	}
+	if len(examples) > 0 {
+		param.Examples = exampleMap(examples)
+	}
+	return param
+}
+
+// marshalToYAMLNode marshals a value using jsoniter (production marshaling) and converts to yaml.Node.
+// The result is an inline JSON representation that preserves integer types for timestamps.
+func marshalToYAMLNode(v any) *yaml.Node {
+	jsonAPI := jsoniter.ConfigCompatibleWithStandardLibrary
+	jsonBytes, err := jsonAPI.Marshal(v)
+	if err != nil {
+		panic(err)
+	}
+	node := &yaml.Node{}
+	if err := yaml.Unmarshal(jsonBytes, node); err != nil {
+		panic(err)
+	}
+	return node
+}
+
+// vectorExample creates an example for a vector query response using production marshaling.
+func vectorExample(v promql.Vector) *yaml.Node {
+	type response struct {
+		Status string `json:"status"`
+		Data   struct {
+			ResultType string        `json:"resultType"`
+			Result     promql.Vector `json:"result"`
+		} `json:"data"`
+	}
+	resp := response{Status: "success"}
+	resp.Data.ResultType = "vector"
+	resp.Data.Result = v
+	return marshalToYAMLNode(resp)
+}
+
+// matrixExample creates an example for a matrix query response using production marshaling.
+func matrixExample(m promql.Matrix) *yaml.Node {
+	type response struct {
+		Status string `json:"status"`
+		Data   struct {
+			ResultType string        `json:"resultType"`
+			Result     promql.Matrix `json:"result"`
+		} `json:"data"`
+	}
+	resp := response{Status: "success"}
+	resp.Data.ResultType = "matrix"
+	resp.Data.Result = m
+	return marshalToYAMLNode(resp)
+}

web/api/v1/openapi_paths.go

@@ -0,0 +1,626 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file defines all API path specifications including parameters, request bodies,
+// and response schemas. Each path definition corresponds to an endpoint registered in api.go.
+package v1
+
+import (
+	"time"
+
+	"github.com/pb33f/libopenapi/datamodel/high/base"
+	v3 "github.com/pb33f/libopenapi/datamodel/high/v3"
+	"github.com/pb33f/libopenapi/orderedmap"
+)
+
+// Path definition methods for API endpoints.
+
+func (*OpenAPIBuilder) queryPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("limit", "The maximum number of metrics to return.", false, integerSchema(), []example{{"example", 100}}),
+		queryParamWithExample("time", "The evaluation timestamp (optional, defaults to current time).", false, timestampSchema(), timestampExamples(exampleTime)),
+		queryParamWithExample("query", "The PromQL query to execute.", true, stringSchema(), []example{{"example", "up"}}),
+		queryParamWithExample("timeout", "Evaluation timeout. Optional. Defaults to and is capped by the value of the -query.timeout flag.", false, stringSchema(), []example{{"example", "30s"}}),
+		queryParamWithExample("lookback_delta", "Override the lookback period for this query. Optional.", false, stringSchema(), []example{{"example", "5m"}}),
+		queryParamWithExample("stats", "When provided, include query statistics in the response. The special value 'all' enables more comprehensive statistics.", false, stringSchema(), []example{{"example", "all"}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "query",
+			Summary:     "Evaluate an instant query",
+			Tags:        []string{"query"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("QueryOutputBody", queryResponseExamples(), errorResponseExamples(), "Query executed successfully.", "Error executing query."),
+		},
+		Post: &v3.Operation{
+			OperationId: "query-post",
+			Summary:     "Evaluate an instant query",
+			Tags:        []string{"query"},
+			RequestBody: formRequestBodyWithExamples("QueryPostInputBody", queryPostExamples(), "Submit an instant query. This endpoint accepts the same parameters as the GET version."),
+			Responses:   responsesWithErrorExamples("QueryOutputBody", queryResponseExamples(), errorResponseExamples(), "Instant query executed successfully.", "Error executing instant query."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) queryRangePath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("limit", "The maximum number of metrics to return.", false, integerSchema(), []example{{"example", 100}}),
+		queryParamWithExample("start", "The start time of the query.", true, timestampSchema(), timestampExamples(exampleTime.Add(-1*time.Hour))),
+		queryParamWithExample("end", "The end time of the query.", true, timestampSchema(), timestampExamples(exampleTime)),
+		queryParamWithExample("step", "The step size of the query.", true, stringSchema(), []example{{"example", "15s"}}),
+		queryParamWithExample("query", "The query to execute.", true, stringSchema(), []example{{"example", "rate(prometheus_http_requests_total{handler=\"/api/v1/query\"}[5m])"}}),
+		queryParamWithExample("timeout", "Evaluation timeout. Optional. Defaults to and is capped by the value of the -query.timeout flag.", false, stringSchema(), []example{{"example", "30s"}}),
+		queryParamWithExample("lookback_delta", "Override the lookback period for this query. Optional.", false, stringSchema(), []example{{"example", "5m"}}),
+		queryParamWithExample("stats", "When provided, include query statistics in the response. The special value 'all' enables more comprehensive statistics.", false, stringSchema(), []example{{"example", "all"}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "query-range",
+			Summary:     "Evaluate a range query",
+			Tags:        []string{"query"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("QueryRangeOutputBody", queryRangeResponseExamples(), errorResponseExamples(), "Range query executed successfully.", "Error executing range query."),
+		},
+		Post: &v3.Operation{
+			OperationId: "query-range-post",
+			Summary:     "Evaluate a range query",
+			Tags:        []string{"query"},
+			RequestBody: formRequestBodyWithExamples("QueryRangePostInputBody", queryRangePostExamples(), "Submit a range query. This endpoint accepts the same parameters as the GET version."),
+			Responses:   responsesWithErrorExamples("QueryRangeOutputBody", queryRangeResponseExamples(), errorResponseExamples(), "Range query executed successfully.", "Error executing range query."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) queryExemplarsPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("start", "Start timestamp for exemplars query.", false, timestampSchema(), timestampExamples(exampleTime.Add(-1*time.Hour))),
+		queryParamWithExample("end", "End timestamp for exemplars query.", false, timestampSchema(), timestampExamples(exampleTime)),
+		queryParamWithExample("query", "PromQL query to extract exemplars for.", true, stringSchema(), []example{{"example", "prometheus_http_requests_total"}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "query-exemplars",
+			Summary:     "Query exemplars",
+			Tags:        []string{"query"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("QueryExemplarsOutputBody", queryExemplarsResponseExamples(), errorResponseExamples(), "Exemplars retrieved successfully.", "Error retrieving exemplars."),
+		},
+		Post: &v3.Operation{
+			OperationId: "query-exemplars-post",
+			Summary:     "Query exemplars",
+			Tags:        []string{"query"},
+			RequestBody: formRequestBodyWithExamples("QueryExemplarsPostInputBody", queryExemplarsPostExamples(), "Submit an exemplars query. This endpoint accepts the same parameters as the GET version."),
+			Responses:   responsesWithErrorExamples("QueryExemplarsOutputBody", queryExemplarsResponseExamples(), errorResponseExamples(), "Exemplars query completed successfully.", "Error processing exemplars query."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) formatQueryPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("query", "PromQL expression to format.", true, stringSchema(), []example{{"example", "sum(rate(http_requests_total[5m])) by (job)"}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "format-query",
+			Summary:     "Format a PromQL query",
+			Tags:        []string{"query"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("FormatQueryOutputBody", formatQueryResponseExamples(), errorResponseExamples(), "Query formatted successfully.", "Error formatting query."),
+		},
+		Post: &v3.Operation{
+			OperationId: "format-query-post",
+			Summary:     "Format a PromQL query",
+			Tags:        []string{"query"},
+			RequestBody: formRequestBodyWithExamples("FormatQueryPostInputBody", formatQueryPostExamples(), "Submit a PromQL query to format. This endpoint accepts the same parameters as the GET version."),
+			Responses:   responsesWithErrorExamples("FormatQueryOutputBody", formatQueryResponseExamples(), errorResponseExamples(), "Query formatting completed successfully.", "Error formatting query."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) parseQueryPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("query", "PromQL expression to parse.", true, stringSchema(), []example{{"example", "up{job=\"prometheus\"}"}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "parse-query",
+			Summary:     "Parse a PromQL query",
+			Tags:        []string{"query"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("ParseQueryOutputBody", parseQueryResponseExamples(), errorResponseExamples(), "Query parsed successfully.", "Error parsing query."),
+		},
+		Post: &v3.Operation{
+			OperationId: "parse-query-post",
+			Summary:     "Parse a PromQL query",
+			Tags:        []string{"query"},
+			RequestBody: formRequestBodyWithExamples("ParseQueryPostInputBody", parseQueryPostExamples(), "Submit a PromQL query to parse. This endpoint accepts the same parameters as the GET version."),
+			Responses:   responsesWithErrorExamples("ParseQueryOutputBody", parseQueryResponseExamples(), errorResponseExamples(), "Query parsed successfully via POST.", "Error parsing query via POST."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) labelsPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("start", "Start timestamp for label names query.", false, timestampSchema(), timestampExamples(exampleTime.Add(-1*time.Hour))),
+		queryParamWithExample("end", "End timestamp for label names query.", false, timestampSchema(), timestampExamples(exampleTime)),
+		queryParamWithExample("match[]", "Series selector argument.", false, base.CreateSchemaProxy(&base.Schema{
+			Type:  []string{"array"},
+			Items: &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		}), []example{{"example", []string{"{job=\"prometheus\"}"}}}),
+		queryParamWithExample("limit", "Maximum number of label names to return.", false, integerSchema(), []example{{"example", 100}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "labels",
+			Summary:     "Get label names",
+			Tags:        []string{"labels"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("LabelsOutputBody", labelsResponseExamples(), errorResponseExamples(), "Label names retrieved successfully.", "Error retrieving label names."),
+		},
+		Post: &v3.Operation{
+			OperationId: "labels-post",
+			Summary:     "Get label names",
+			Tags:        []string{"labels"},
+			RequestBody: formRequestBodyWithExamples("LabelsPostInputBody", labelsPostExamples(), "Submit a label names query. This endpoint accepts the same parameters as the GET version."),
+			Responses:   responsesWithErrorExamples("LabelsOutputBody", labelsResponseExamples(), errorResponseExamples(), "Label names retrieved successfully via POST.", "Error retrieving label names via POST."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) labelValuesPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		pathParam("name", "Label name.", stringSchema()),
+		queryParamWithExample("start", "Start timestamp for label values query.", false, timestampSchema(), timestampExamples(exampleTime.Add(-1*time.Hour))),
+		queryParamWithExample("end", "End timestamp for label values query.", false, timestampSchema(), timestampExamples(exampleTime)),
+		queryParamWithExample("match[]", "Series selector argument.", false, base.CreateSchemaProxy(&base.Schema{
+			Type:  []string{"array"},
+			Items: &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		}), []example{{"example", []string{"{job=\"prometheus\"}"}}}),
+		queryParamWithExample("limit", "Maximum number of label values to return.", false, integerSchema(), []example{{"example", 1000}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "label-values",
+			Summary:     "Get label values",
+			Tags:        []string{"labels"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("LabelValuesOutputBody", labelValuesResponseExamples(), errorResponseExamples(), "Label values retrieved successfully.", "Error retrieving label values."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) seriesPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("start", "Start timestamp for series query.", false, timestampSchema(), timestampExamples(exampleTime.Add(-1*time.Hour))),
+		queryParamWithExample("end", "End timestamp for series query.", false, timestampSchema(), timestampExamples(exampleTime)),
+		queryParamWithExample("match[]", "Series selector argument.", true, base.CreateSchemaProxy(&base.Schema{
+			Type:  []string{"array"},
+			Items: &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		}), []example{{"example", []string{"{job=\"prometheus\"}"}}}),
+		queryParamWithExample("limit", "Maximum number of series to return.", false, integerSchema(), []example{{"example", 100}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "series",
+			Summary:     "Find series by label matchers",
+			Tags:        []string{"series"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("SeriesOutputBody", seriesResponseExamples(), errorResponseExamples(), "Series returned matching the provided label matchers.", "Error retrieving series."),
+		},
+		Post: &v3.Operation{
+			OperationId: "series-post",
+			Summary:     "Find series by label matchers",
+			Tags:        []string{"series"},
+			RequestBody: formRequestBodyWithExamples("SeriesPostInputBody", seriesPostExamples(), "Submit a series query. This endpoint accepts the same parameters as the GET version."),
+			Responses:   responsesWithErrorExamples("SeriesOutputBody", seriesResponseExamples(), errorResponseExamples(), "Series returned matching the provided label matchers via POST.", "Error retrieving series via POST."),
+		},
+		Delete: &v3.Operation{
+			OperationId: "delete-series",
+			Summary:     "Delete series",
+			Description: "Delete series matching selectors. Note: This is deprecated, use POST /admin/tsdb/delete_series instead.",
+			Tags:        []string{"series"},
+			Responses:   responsesWithErrorExamples("SeriesDeleteOutputBody", seriesDeleteResponseExamples(), errorResponseExamples(), "Series marked for deletion.", "Error deleting series."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) metadataPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("limit", "The maximum number of metrics to return.", false, integerSchema(), []example{{"example", 100}}),
+		queryParamWithExample("limit_per_metric", "The maximum number of metadata entries per metric.", false, integerSchema(), []example{{"example", 10}}),
+		queryParamWithExample("metric", "A metric name to filter metadata for.", false, stringSchema(), []example{{"example", "http_requests_total"}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-metadata",
+			Summary:     "Get metadata",
+			Tags:        []string{"metadata"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("MetadataOutputBody", metadataResponseExamples(), errorResponseExamples(), "Metric metadata retrieved successfully.", "Error retrieving metadata."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) scrapePoolsPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-scrape-pools",
+			Summary:     "Get scrape pools",
+			Tags:        []string{"targets"},
+			Responses:   responsesWithErrorExamples("ScrapePoolsOutputBody", scrapePoolsResponseExamples(), errorResponseExamples(), "Scrape pools retrieved successfully.", "Error retrieving scrape pools."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) targetsPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("scrapePool", "Filter targets by scrape pool name.", false, stringSchema(), []example{{"example", "prometheus"}}),
+		queryParamWithExample("state", "Filter by state: active, dropped, or any.", false, stringSchema(), []example{{"example", "active"}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-targets",
+			Summary:     "Get targets",
+			Tags:        []string{"targets"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("TargetsOutputBody", targetsResponseExamples(), errorResponseExamples(), "Target discovery information retrieved successfully.", "Error retrieving targets."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) targetsMetadataPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("match_target", "Label selector to filter targets.", false, stringSchema(), []example{{"example", "{job=\"prometheus\"}"}}),
+		queryParamWithExample("metric", "Metric name to retrieve metadata for.", false, stringSchema(), []example{{"example", "http_requests_total"}}),
+		queryParamWithExample("limit", "Maximum number of targets to match.", false, integerSchema(), []example{{"example", 10}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-targets-metadata",
+			Summary:     "Get targets metadata",
+			Tags:        []string{"targets"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("TargetMetadataOutputBody", targetsMetadataResponseExamples(), errorResponseExamples(), "Target metadata retrieved successfully.", "Error retrieving target metadata."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) targetsRelabelStepsPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("scrapePool", "Name of the scrape pool.", true, stringSchema(), []example{{"example", "prometheus"}}),
+		queryParamWithExample("labels", "JSON-encoded labels to apply relabel rules to.", true, stringSchema(), []example{{"example", "{\"__address__\":\"localhost:9090\",\"job\":\"prometheus\"}"}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-targets-relabel-steps",
+			Summary:     "Get targets relabel steps",
+			Tags:        []string{"targets"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("TargetRelabelStepsOutputBody", targetsRelabelStepsResponseExamples(), errorResponseExamples(), "Relabel steps retrieved successfully.", "Error retrieving relabel steps."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) rulesPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("type", "Filter by rule type: alert or record.", false, stringSchema(), []example{{"example", "alert"}}),
+		queryParamWithExample("rule_name[]", "Filter by rule name.", false, base.CreateSchemaProxy(&base.Schema{
+			Type:  []string{"array"},
+			Items: &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		}), []example{{"example", []string{"HighErrorRate"}}}),
+		queryParamWithExample("rule_group[]", "Filter by rule group name.", false, base.CreateSchemaProxy(&base.Schema{
+			Type:  []string{"array"},
+			Items: &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		}), []example{{"example", []string{"example_alerts"}}}),
+		queryParamWithExample("file[]", "Filter by file path.", false, base.CreateSchemaProxy(&base.Schema{
+			Type:  []string{"array"},
+			Items: &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		}), []example{{"example", []string{"/etc/prometheus/rules.yml"}}}),
+		queryParamWithExample("match[]", "Label matchers to filter rules.", false, base.CreateSchemaProxy(&base.Schema{
+			Type:  []string{"array"},
+			Items: &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		}), []example{{"example", []string{"{severity=\"critical\"}"}}}),
+		queryParamWithExample("exclude_alerts", "Exclude active alerts from response.", false, stringSchema(), []example{{"example", "false"}}),
+		queryParamWithExample("group_limit", "Maximum number of rule groups to return.", false, integerSchema(), []example{{"example", 100}}),
+		queryParamWithExample("group_next_token", "Pagination token for next page.", false, stringSchema(), []example{{"example", "abc123"}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "rules",
+			Summary:     "Get alerting and recording rules",
+			Tags:        []string{"rules"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("RulesOutputBody", rulesResponseExamples(), errorResponseExamples(), "Rules retrieved successfully.", "Error retrieving rules."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) alertsPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "alerts",
+			Summary:     "Get active alerts",
+			Tags:        []string{"alerts"},
+			Responses:   responsesWithErrorExamples("AlertsOutputBody", alertsResponseExamples(), errorResponseExamples(), "Active alerts retrieved successfully.", "Error retrieving alerts."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) alertmanagersPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "alertmanagers",
+			Summary:     "Get Alertmanager discovery",
+			Tags:        []string{"alerts"},
+			Responses:   responsesWithErrorExamples("AlertmanagersOutputBody", alertmanagersResponseExamples(), errorResponseExamples(), "Alertmanager targets retrieved successfully.", "Error retrieving Alertmanager targets."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) statusConfigPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-status-config",
+			Summary:     "Get status config",
+			Tags:        []string{"status"},
+			Responses:   responsesWithErrorExamples("StatusConfigOutputBody", statusConfigResponseExamples(), errorResponseExamples(), "Configuration retrieved successfully.", "Error retrieving configuration."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) statusRuntimeInfoPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-status-runtimeinfo",
+			Summary:     "Get status runtimeinfo",
+			Tags:        []string{"status"},
+			Responses:   responsesWithErrorExamples("StatusRuntimeInfoOutputBody", statusRuntimeInfoResponseExamples(), errorResponseExamples(), "Runtime information retrieved successfully.", "Error retrieving runtime information."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) statusBuildInfoPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-status-buildinfo",
+			Summary:     "Get status buildinfo",
+			Tags:        []string{"status"},
+			Responses:   responsesWithErrorExamples("StatusBuildInfoOutputBody", statusBuildInfoResponseExamples(), errorResponseExamples(), "Build information retrieved successfully.", "Error retrieving build information."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) statusFlagsPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-status-flags",
+			Summary:     "Get status flags",
+			Tags:        []string{"status"},
+			Responses:   responsesWithErrorExamples("StatusFlagsOutputBody", statusFlagsResponseExamples(), errorResponseExamples(), "Command-line flags retrieved successfully.", "Error retrieving flags."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) statusTSDBPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("limit", "The maximum number of items to return per category.", false, integerSchema(), []example{{"example", 10}}),
+	}
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "status-tsdb",
+			Summary:     "Get TSDB status",
+			Tags:        []string{"status"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("StatusTSDBOutputBody", statusTSDBResponseExamples(), errorResponseExamples(), "TSDB status retrieved successfully.", "Error retrieving TSDB status."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) statusTSDBBlocksPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "status-tsdb-blocks",
+			Summary:     "Get TSDB blocks information",
+			Tags:        []string{"status"},
+			Responses:   responsesWithErrorExamples("StatusTSDBBlocksOutputBody", statusTSDBBlocksResponseExamples(), errorResponseExamples(), "TSDB blocks information retrieved successfully.", "Error retrieving TSDB blocks."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) statusWALReplayPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-status-walreplay",
+			Summary:     "Get status walreplay",
+			Tags:        []string{"status"},
+			Responses:   responsesWithErrorExamples("StatusWALReplayOutputBody", statusWALReplayResponseExamples(), errorResponseExamples(), "WAL replay status retrieved successfully.", "Error retrieving WAL replay status."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) adminDeleteSeriesPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("match[]", "Series selectors to identify series to delete.", true, base.CreateSchemaProxy(&base.Schema{
+			Type:  []string{"array"},
+			Items: &base.DynamicValue[*base.SchemaProxy, bool]{A: stringSchema()},
+		}), []example{{"example", []string{"{__name__=~\"test.*\"}"}}}),
+		queryParamWithExample("start", "Start timestamp for deletion.", false, timestampSchema(), timestampExamples(exampleTime.Add(-1*time.Hour))),
+		queryParamWithExample("end", "End timestamp for deletion.", false, timestampSchema(), timestampExamples(exampleTime)),
+	}
+	return &v3.PathItem{
+		Post: &v3.Operation{
+			OperationId: "deleteSeriesPost",
+			Summary:     "Delete series matching selectors",
+			Description: "Deletes data for a selection of series in a time range.",
+			Tags:        []string{"admin"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("DeleteSeriesOutputBody", deleteSeriesResponseExamples(), errorResponseExamples(), "Series deleted successfully.", "Error deleting series."),
+		},
+		Put: &v3.Operation{
+			OperationId: "deleteSeriesPut",
+			Summary:     "Delete series matching selectors via PUT",
+			Description: "Deletes data for a selection of series in a time range using PUT method.",
+			Tags:        []string{"admin"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("DeleteSeriesOutputBody", deleteSeriesResponseExamples(), errorResponseExamples(), "Series deleted successfully via PUT.", "Error deleting series via PUT."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) adminCleanTombstonesPath() *v3.PathItem {
+	return &v3.PathItem{
+		Post: &v3.Operation{
+			OperationId: "cleanTombstonesPost",
+			Summary:     "Clean tombstones in the TSDB",
+			Description: "Removes deleted data from disk and cleans up existing tombstones.",
+			Tags:        []string{"admin"},
+			Responses:   responsesWithErrorExamples("CleanTombstonesOutputBody", cleanTombstonesResponseExamples(), errorResponseExamples(), "Tombstones cleaned successfully.", "Error cleaning tombstones."),
+		},
+		Put: &v3.Operation{
+			OperationId: "cleanTombstonesPut",
+			Summary:     "Clean tombstones in the TSDB via PUT",
+			Description: "Removes deleted data from disk and cleans up existing tombstones using PUT method.",
+			Tags:        []string{"admin"},
+			Responses:   responsesWithErrorExamples("CleanTombstonesOutputBody", cleanTombstonesResponseExamples(), errorResponseExamples(), "Tombstones cleaned successfully via PUT.", "Error cleaning tombstones via PUT."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) adminSnapshotPath() *v3.PathItem {
+	params := []*v3.Parameter{
+		queryParamWithExample("skip_head", "If true, do not snapshot data in the head block.", false, stringSchema(), []example{{"example", "false"}}),
+	}
+	return &v3.PathItem{
+		Post: &v3.Operation{
+			OperationId: "snapshotPost",
+			Summary:     "Create a snapshot of the TSDB",
+			Description: "Creates a snapshot of all current data.",
+			Tags:        []string{"admin"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("SnapshotOutputBody", snapshotResponseExamples(), errorResponseExamples(), "Snapshot created successfully.", "Error creating snapshot."),
+		},
+		Put: &v3.Operation{
+			OperationId: "snapshotPut",
+			Summary:     "Create a snapshot of the TSDB via PUT",
+			Description: "Creates a snapshot of all current data using PUT method.",
+			Tags:        []string{"admin"},
+			Parameters:  params,
+			Responses:   responsesWithErrorExamples("SnapshotOutputBody", snapshotResponseExamples(), errorResponseExamples(), "Snapshot created successfully via PUT.", "Error creating snapshot via PUT."),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) remoteReadPath() *v3.PathItem {
+	return &v3.PathItem{
+		Post: &v3.Operation{
+			OperationId: "remoteRead",
+			Summary:     "Remote read endpoint",
+			Description: "Prometheus remote read endpoint for federated queries. Accepts and returns Protocol Buffer encoded data.",
+			Tags:        []string{"remote"},
+			Responses:   responsesNoContent(),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) remoteWritePath() *v3.PathItem {
+	return &v3.PathItem{
+		Post: &v3.Operation{
+			OperationId: "remoteWrite",
+			Summary:     "Remote write endpoint",
+			Description: "Prometheus remote write endpoint for sending metrics. Accepts Protocol Buffer encoded write requests.",
+			Tags:        []string{"remote"},
+			Responses:   responsesNoContent(),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) otlpWritePath() *v3.PathItem {
+	return &v3.PathItem{
+		Post: &v3.Operation{
+			OperationId: "otlpWrite",
+			Summary:     "OTLP metrics write endpoint",
+			Description: "OpenTelemetry Protocol metrics ingestion endpoint. Accepts OTLP/HTTP metrics in Protocol Buffer format.",
+			Tags:        []string{"otlp"},
+			Responses:   responsesNoContent(),
+		},
+	}
+}
+
+func (*OpenAPIBuilder) notificationsPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-notifications",
+			Summary:     "Get notifications",
+			Tags:        []string{"notifications"},
+			Responses:   responsesWithErrorExamples("NotificationsOutputBody", notificationsResponseExamples(), errorResponseExamples(), "Notifications retrieved successfully.", "Error retrieving notifications."),
+		},
+	}
+}
+
+// notificationsLivePath defines the /notifications/live endpoint.
+// This endpoint uses OpenAPI 3.2's itemSchema feature for documenting SSE streams.
+// It is excluded from the OpenAPI 3.1 specification.
+func (*OpenAPIBuilder) notificationsLivePath() *v3.PathItem {
+	codes := orderedmap.New[string, *v3.Response]()
+	content := orderedmap.New[string, *v3.MediaType]()
+
+	// Create a schema for the SSE message structure.
+	// Each SSE message has a 'data' field containing JSON.
+	sseItemProps := orderedmap.New[string, *base.SchemaProxy]()
+	sseItemProps.Set("data", base.CreateSchemaProxy(&base.Schema{
+		Type:             []string{"string"},
+		Description:      "SSE data field containing JSON-encoded notification.",
+		ContentMediaType: "application/json",
+		ContentSchema:    schemaRef("#/components/schemas/Notification"),
+	}))
+
+	content.Set("text/event-stream", &v3.MediaType{
+		// Use ItemSchema (OpenAPI 3.2) instead of Schema to describe each SSE message.
+		ItemSchema: base.CreateSchemaProxy(&base.Schema{
+			Type:                 []string{"object"},
+			Title:                "Server Sent Event Message",
+			Description:          "A single SSE message. The data field contains a JSON-encoded Notification object.",
+			Properties:           sseItemProps,
+			Required:             []string{"data"},
+			AdditionalProperties: &base.DynamicValue[*base.SchemaProxy, bool]{N: 1, B: false},
+		}),
+		Examples: notificationLiveExamples(),
+	})
+
+	codes.Set("200", &v3.Response{
+		Description: "Server-sent events stream established.",
+		Content:     content,
+	})
+	codes.Set("default", errorResponse())
+
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "notifications-live",
+			Summary:     "Stream live notifications via Server-Sent Events",
+			Description: "Subscribe to real-time server notifications using SSE. Each event contains a JSON-encoded Notification object in the data field.",
+			Tags:        []string{"notifications"},
+			Responses:   &v3.Responses{Codes: codes},
+		},
+	}
+}
+
+func (*OpenAPIBuilder) featuresPath() *v3.PathItem {
+	return &v3.PathItem{
+		Get: &v3.Operation{
+			OperationId: "get-features",
+			Summary:     "Get features",
+			Tags:        []string{"features"},
+			Responses:   responsesWithErrorExamples("FeaturesOutputBody", featuresResponseExamples(), errorResponseExamples(), "Feature flags retrieved successfully.", "Error retrieving features."),
+		},
+	}
+}

web/api/v1/openapi_test.go

@@ -0,0 +1,289 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
+)
+
+// TestOpenAPIHTTPHandler verifies that the OpenAPI endpoint serves a valid specification
+// with correct headers, structure conforming to OpenAPI 3.1 standards, and consistent responses.
+func TestOpenAPIHTTPHandler(t *testing.T) {
+	builder := NewOpenAPIBuilder(OpenAPIOptions{}, promslog.NewNopLogger())
+
+	// First request.
+	req1 := httptest.NewRequest(http.MethodGet, "/api/v1/openapi.yaml", nil)
+	rec1 := httptest.NewRecorder()
+	builder.ServeOpenAPI(rec1, req1)
+
+	// Verify status code and headers.
+	require.Equal(t, http.StatusOK, rec1.Code)
+	require.True(t, strings.HasPrefix(rec1.Header().Get("Content-Type"), "application/yaml"), "Content-Type should start with application/yaml")
+	require.Equal(t, "no-cache, no-store, must-revalidate", rec1.Header().Get("Cache-Control"))
+
+	// Verify it is valid YAML.
+	var spec map[string]any
+	err := yaml.Unmarshal(rec1.Body.Bytes(), &spec)
+	require.NoError(t, err)
+
+	// Verify structure.
+	require.Contains(t, spec, "openapi")
+	require.Contains(t, spec, "info")
+	require.Contains(t, spec, "paths")
+	require.Contains(t, spec, "components")
+
+	// Verify OpenAPI version (default is 3.1.0).
+	require.Equal(t, "3.1.0", spec["openapi"])
+
+	// Verify info section.
+	info, ok := spec["info"].(map[any]any)
+	require.True(t, ok, "info should be a map")
+	require.Equal(t, "Prometheus API", info["title"])
+
+	// Verify paths exist.
+	paths, ok := spec["paths"].(map[any]any)
+	require.True(t, ok, "paths should be a map")
+	require.NotEmpty(t, paths, "paths should not be empty")
+
+	// Second request to verify response consistency.
+	req2 := httptest.NewRequest(http.MethodGet, "/api/v1/openapi.yaml", nil)
+	rec2 := httptest.NewRecorder()
+	builder.ServeOpenAPI(rec2, req2)
+
+	// Both responses should be identical.
+	require.Equal(t, rec1.Body.String(), rec2.Body.String())
+}
+
+// TestOpenAPIPathFiltering verifies that the IncludePaths option correctly filters
+// which API paths are included in the generated specification.
+func TestOpenAPIPathFiltering(t *testing.T) {
+	tests := []struct {
+		name         string
+		includePaths []string
+		wantPaths    []string
+		excludePaths []string
+	}{
+		{
+			name:         "no filter includes all",
+			includePaths: nil,
+			wantPaths:    []string{"/query", "/labels", "/alerts", "/targets"},
+		},
+		{
+			name:         "filter query paths",
+			includePaths: []string{"/query"},
+			wantPaths:    []string{"/query", "/query_range", "/query_exemplars"},
+			excludePaths: []string{"/labels", "/alerts", "/targets"},
+		},
+		{
+			name:         "filter status paths",
+			includePaths: []string{"/status"},
+			wantPaths:    []string{"/status/config", "/status/flags", "/status/runtimeinfo"},
+			excludePaths: []string{"/query", "/alerts", "/targets"},
+		},
+		{
+			name:         "filter multiple prefixes",
+			includePaths: []string{"/label", "/series"},
+			wantPaths:    []string{"/labels", "/label/{name}/values", "/series"},
+			excludePaths: []string{"/query", "/alerts", "/targets"},
+		},
+		{
+			name:         "exact path match",
+			includePaths: []string{"/alerts"},
+			wantPaths:    []string{"/alerts"},
+			excludePaths: []string{"/alertmanagers", "/query"},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			builder := NewOpenAPIBuilder(OpenAPIOptions{
+				IncludePaths: tc.includePaths,
+			}, promslog.NewNopLogger())
+
+			req := httptest.NewRequest(http.MethodGet, "/api/v1/openapi.yaml", nil)
+			rec := httptest.NewRecorder()
+			builder.ServeOpenAPI(rec, req)
+
+			require.Equal(t, http.StatusOK, rec.Code)
+
+			var spec map[string]any
+			err := yaml.Unmarshal(rec.Body.Bytes(), &spec)
+			require.NoError(t, err)
+
+			paths, ok := spec["paths"].(map[any]any)
+			require.True(t, ok, "paths should be a map")
+
+			for _, want := range tc.wantPaths {
+				require.Contains(t, paths, want)
+			}
+
+			for _, exclude := range tc.excludePaths {
+				require.NotContains(t, paths, exclude)
+			}
+		})
+	}
+}
+
+// TestOpenAPISchemaCompleteness verifies that all referenced schemas in paths
+// are defined in the components/schemas section of the specification.
+func TestOpenAPISchemaCompleteness(t *testing.T) {
+	builder := NewOpenAPIBuilder(OpenAPIOptions{}, promslog.NewNopLogger())
+
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/openapi.yaml", nil)
+	rec := httptest.NewRecorder()
+	builder.ServeOpenAPI(rec, req)
+
+	var spec map[string]any
+	err := yaml.Unmarshal(rec.Body.Bytes(), &spec)
+	require.NoError(t, err)
+
+	components, ok := spec["components"].(map[any]any)
+	require.True(t, ok, "components should be a map")
+
+	schemas, ok := components["schemas"].(map[any]any)
+	require.True(t, ok, "schemas should be a map")
+
+	// Verify essential schemas are present.
+	essentialSchemas := []string{
+		"Error",
+		"Labels",
+		"QueryOutputBody",
+		"LabelsOutputBody",
+		"SeriesOutputBody",
+		"TargetsOutputBody",
+		"AlertsOutputBody",
+		"RulesOutputBody",
+		"StatusConfigOutputBody",
+		"StatusFlagsOutputBody",
+		"PrometheusVersion",
+	}
+
+	for _, schema := range essentialSchemas {
+		require.Contains(t, schemas, schema)
+	}
+}
+
+// TODO: Add test to verify all routes from api.go Register() are covered in OpenAPI spec.
+// Consider wrapping Router to track registered paths and cross-check with OpenAPI paths.
+
+// TestOpenAPIShouldIncludePath verifies the shouldIncludePath method correctly
+// matches paths against the IncludePaths filter configuration.
+func TestOpenAPIShouldIncludePath(t *testing.T) {
+	tests := []struct {
+		name         string
+		includePaths []string
+		path         string
+		expected     bool
+	}{
+		{
+			name:         "empty filter includes all",
+			includePaths: nil,
+			path:         "/query",
+			expected:     true,
+		},
+		{
+			name:         "exact match",
+			includePaths: []string{"/query"},
+			path:         "/query",
+			expected:     true,
+		},
+		{
+			name:         "prefix match",
+			includePaths: []string{"/query"},
+			path:         "/query_range",
+			expected:     true,
+		},
+		{
+			name:         "no match",
+			includePaths: []string{"/query"},
+			path:         "/labels",
+			expected:     false,
+		},
+		{
+			name:         "multiple filters with match",
+			includePaths: []string{"/labels", "/series"},
+			path:         "/series",
+			expected:     true,
+		},
+		{
+			name:         "multiple filters without match",
+			includePaths: []string{"/labels", "/series"},
+			path:         "/query",
+			expected:     false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			builder := &OpenAPIBuilder{
+				options: OpenAPIOptions{
+					IncludePaths: tc.includePaths,
+				},
+			}
+
+			result := builder.shouldIncludePath(tc.path)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+// TestOpenAPIVersionConsistency verifies that both OpenAPI versions are properly generated
+// and that 3.2 has exactly one more path than 3.1 (/notifications/live).
+func TestOpenAPIVersionConsistency(t *testing.T) {
+	builder := NewOpenAPIBuilder(OpenAPIOptions{}, promslog.NewNopLogger())
+
+	// Fetch OpenAPI 3.1 spec (default).
+	req31 := httptest.NewRequest(http.MethodGet, "/api/v1/openapi.yaml", nil)
+	rec31 := httptest.NewRecorder()
+	builder.ServeOpenAPI(rec31, req31)
+
+	require.Equal(t, http.StatusOK, rec31.Code)
+
+	// Fetch OpenAPI 3.2 spec.
+	req32 := httptest.NewRequest(http.MethodGet, "/api/v1/openapi.yaml?openapi_version=3.2", nil)
+	rec32 := httptest.NewRecorder()
+	builder.ServeOpenAPI(rec32, req32)
+
+	require.Equal(t, http.StatusOK, rec32.Code)
+
+	// Parse both specs.
+	var spec31, spec32 map[string]any
+	err := yaml.Unmarshal(rec31.Body.Bytes(), &spec31)
+	require.NoError(t, err)
+	err = yaml.Unmarshal(rec32.Body.Bytes(), &spec32)
+	require.NoError(t, err)
+
+	// Verify versions are different.
+	require.Equal(t, "3.1.0", spec31["openapi"])
+	require.Equal(t, "3.2.0", spec32["openapi"])
+
+	// Verify /notifications/live is only in 3.2.
+	paths31 := spec31["paths"].(map[any]any)
+	paths32 := spec32["paths"].(map[any]any)
+
+	require.NotContains(t, paths31, "/notifications/live")
+
+	require.Contains(t, paths32, "/notifications/live")
+
+	// Verify 3.2 has exactly one more path than 3.1.
+	require.Len(t, paths32, len(paths31)+1,
+		"OpenAPI 3.2 should have exactly one more path than 3.1")
+}

web/api/v1/test_helpers.go

@@ -0,0 +1,157 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/prometheus/common/route"
+
+	"github.com/prometheus/prometheus/web/api/testhelpers"
+)
+
+// newTestAPI creates a new API instance for testing using testhelpers.
+func newTestAPI(t *testing.T, cfg testhelpers.APIConfig) *testhelpers.APIWrapper {
+	t.Helper()
+
+	params := testhelpers.PrepareAPI(t, cfg)
+
+	// Adapt the testhelpers interfaces to v1 interfaces.
+	api := NewAPI(
+		params.QueryEngine,
+		params.Queryable,
+		nil, // appendable
+		params.ExemplarQueryable,
+		func(ctx context.Context) ScrapePoolsRetriever {
+			return adaptScrapePoolsRetriever(params.ScrapePoolsRetriever(ctx))
+		},
+		func(ctx context.Context) TargetRetriever {
+			return adaptTargetRetriever(params.TargetRetriever(ctx))
+		},
+		func(ctx context.Context) AlertmanagerRetriever {
+			return adaptAlertmanagerRetriever(params.AlertmanagerRetriever(ctx))
+		},
+		params.ConfigFunc,
+		params.FlagsMap,
+		GlobalURLOptions{},
+		params.ReadyFunc,
+		adaptTSDBAdminStats(params.TSDBAdmin),
+		params.DBDir,
+		false, // enableAdmin
+		params.Logger,
+		func(ctx context.Context) RulesRetriever {
+			return adaptRulesRetriever(params.RulesRetriever(ctx))
+		},
+		0,     // remoteReadSampleLimit
+		0,     // remoteReadConcurrencyLimit
+		0,     // remoteReadMaxBytesInFrame
+		false, // isAgent
+		nil,   // corsOrigin
+		func() (RuntimeInfo, error) {
+			info, err := params.RuntimeInfoFunc()
+			return RuntimeInfo{
+				StartTime:           info.StartTime,
+				CWD:                 info.CWD,
+				Hostname:            info.Hostname,
+				ServerTime:          info.ServerTime,
+				ReloadConfigSuccess: info.ReloadConfigSuccess,
+				LastConfigTime:      info.LastConfigTime,
+				CorruptionCount:     info.CorruptionCount,
+				GoroutineCount:      info.GoroutineCount,
+				GOMAXPROCS:          info.GOMAXPROCS,
+				GOMEMLIMIT:          info.GOMEMLIMIT,
+				GOGC:                info.GOGC,
+				GODEBUG:             info.GODEBUG,
+				StorageRetention:    info.StorageRetention,
+			}, err
+		},
+		&PrometheusVersion{
+			Version:   params.BuildInfo.Version,
+			Revision:  params.BuildInfo.Revision,
+			Branch:    params.BuildInfo.Branch,
+			BuildUser: params.BuildInfo.BuildUser,
+			BuildDate: params.BuildInfo.BuildDate,
+			GoVersion: params.BuildInfo.GoVersion,
+		},
+		params.NotificationsGetter,
+		params.NotificationsSub,
+		params.Gatherer,
+		params.Registerer,
+		nil,              // statsRenderer
+		false,            // rwEnabled
+		nil,              // acceptRemoteWriteProtoMsgs
+		false,            // otlpEnabled
+		false,            // otlpDeltaToCumulative
+		false,            // otlpNativeDeltaIngestion
+		false,            // stZeroIngestionEnabled
+		5*time.Minute,    // lookbackDelta
+		false,            // enableTypeAndUnitLabels
+		false,            // appendMetadata
+		nil,              // overrideErrorCode
+		nil,              // featureRegistry
+		OpenAPIOptions{}, // openAPIOptions
+	)
+
+	// Register routes.
+	router := route.New()
+	api.Register(router.WithPrefix("/api/v1"))
+
+	return &testhelpers.APIWrapper{
+		Handler: router,
+	}
+}
+
+// Adapter functions to convert testhelpers interfaces to v1 interfaces.
+
+type rulesRetrieverAdapter struct {
+	testhelpers.RulesRetriever
+}
+
+func adaptRulesRetriever(r testhelpers.RulesRetriever) RulesRetriever {
+	return &rulesRetrieverAdapter{r}
+}
+
+type targetRetrieverAdapter struct {
+	testhelpers.TargetRetriever
+}
+
+func adaptTargetRetriever(t testhelpers.TargetRetriever) TargetRetriever {
+	return &targetRetrieverAdapter{t}
+}
+
+type scrapePoolsRetrieverAdapter struct {
+	testhelpers.ScrapePoolsRetriever
+}
+
+func adaptScrapePoolsRetriever(s testhelpers.ScrapePoolsRetriever) ScrapePoolsRetriever {
+	return &scrapePoolsRetrieverAdapter{s}
+}
+
+type alertmanagerRetrieverAdapter struct {
+	testhelpers.AlertmanagerRetriever
+}
+
+func adaptAlertmanagerRetriever(a testhelpers.AlertmanagerRetriever) AlertmanagerRetriever {
+	return &alertmanagerRetrieverAdapter{a}
+}
+
+type tsdbAdminStatsAdapter struct {
+	testhelpers.TSDBAdminStats
+}
+
+func adaptTSDBAdminStats(t testhelpers.TSDBAdminStats) TSDBAdminStats {
+	return &tsdbAdminStatsAdapter{t}
+}

web/federate_test.go

@@ -212,7 +212,6 @@ func TestFederation(t *testing.T) {
 			test_metric_stale                       1+10x99 stale
 			test_metric_old                         1+10x98
 	`)
-	t.Cleanup(func() { storage.Close() })
 
 	h := &Handler{
 		localStorage:  &dbAdapter{storage.DB},
@@ -303,7 +302,6 @@ func normalizeBody(body *bytes.Buffer) string {
 
 func TestFederationWithNativeHistograms(t *testing.T) {
 	storage := teststorage.New(t)
-	t.Cleanup(func() { storage.Close() })
 
 	var expVec promql.Vector
 

web/ui/mantine-ui/src/promql/functionDocs.tsx

@@ -1756,6 +1756,12 @@ const funcDocs: Record<string, React.ReactNode> = {
         .
       </p>
 
+      <p>
+        Note that if there are any time series in <code>v</code> that match the <code>data-label-selector</code> (or the
+        default <code>target_info</code> if that argument is not specified), they will be treated as info series and
+        will be returned unchanged.
+      </p>
+
       <h3>Limitations</h3>
 
       <p>

web/ui/mantine-ui/src/promql/tools/go.mod

@@ -1,6 +1,6 @@
 module github.com/prometheus/prometheus/web/ui/mantine-ui/src/promql/tools
 
-go 1.24.0
+go 1.25.0
 
 require (
 	github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853

web/ui/package-lock.json

@@ -7823,9 +7823,9 @@
       }
     },
     "node_modules/react-router": {
-      "version": "7.9.5",
-      "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.9.5.tgz",
-      "integrity": "sha512-JmxqrnBZ6E9hWmf02jzNn9Jm3UqyeimyiwzD69NjxGySG6lIz/1LVPsoTCwN7NBX2XjCEa1LIX5EMz1j2b6u6A==",
+      "version": "7.12.0",
+      "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.12.0.tgz",
+      "integrity": "sha512-kTPDYPFzDVGIIGNLS5VJykK0HfHLY5MF3b+xj0/tTyNYL1gF1qs7u67Z9jEhQk2sQ98SUaHxlG31g1JtF7IfVw==",
       "dependencies": {
         "cookie": "^1.0.1",
         "set-cookie-parser": "^2.6.0"

web/ui/react-app/src/globals.ts

@@ -1,6 +1,5 @@
 import jquery from 'jquery';
+import moment from 'moment';
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-(window as any).jQuery = jquery;
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-(window as any).moment = require('moment');
+window.jQuery = jquery;
+window.moment = moment;

web/ui/react-app/src/types/index.d.ts

@@ -68,3 +68,8 @@ interface JQueryStatic {
     scale: () => Color;
   };
 }
+
+interface Window {
+  jQuery: JQueryStatic;
+  moment: typeof import('moment');
+}