Merge branch 'main' into codesome/stale-series-compaction

Signed-off-by: Ganesh Vernekar <ganesh.vernekar@reddit.com>

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/setup_environment
         with:
           enable_npm: true
@@ -37,7 +37,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/setup_environment
       - run: go test --tags=dedupelabels ./...
       - run: go test --tags=slicelabels -race ./cmd/prometheus ./model/textparse ./prompb/...
@@ -81,7 +81,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/setup_environment
         with:
           enable_go: false
@@ -146,7 +146,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/build
         with:
           promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386"
@@ -173,7 +173,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/build
         with:
           parallelism: 12
@@ -212,7 +212,7 @@ jobs:
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/setup_environment
         with:
           enable_npm: true
@@ -270,7 +270,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/publish_main
         with:
           docker_hub_login: ${{ secrets.docker_hub_login }}
@@ -289,7 +289,7 @@ jobs:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - uses: ./.github/promci/actions/publish_release
         with:
           docker_hub_login: ${{ secrets.docker_hub_login }}
@@ -306,7 +306,7 @@ jobs:
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           persist-credentials: false
-      - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3
+      - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4
       - name: Install nodejs
         uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
         with:

.github/workflows/fuzzing.yml

@@ -1,30 +1,47 @@
-name: CIFuzz
+name: fuzzing
 on:
   workflow_call:
 permissions:
   contents: read
 
 jobs:
-  Fuzzing:
+  fuzzing:
+    name: Run Go Fuzz Tests
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        fuzz_test: [FuzzParseMetricText, FuzzParseOpenMetric, FuzzParseMetricSelector, FuzzParseExpr]
     steps:
-      - name: Build Fuzzers
-        id: build
-        uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@4bf20ff8dfda18ad651583ebca9fb17a7ce1940a # master
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          oss-fuzz-project-name: "prometheus"
-          dry-run: false
-      - name: Run Fuzzers
-        uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@4bf20ff8dfda18ad651583ebca9fb17a7ce1940a # master
-        # Note: Regularly check for updates to the pinned commit hash at:
-        # https://github.com/google/oss-fuzz/tree/master/infra/cifuzz/actions/run_fuzzers
+          persist-credentials: false
+      - name: Install Go
+        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
         with:
-          oss-fuzz-project-name: "prometheus"
-          fuzz-seconds: 600
-          dry-run: false
-      - name: Upload Crash
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
-        if: failure() && steps.build.outcome == 'success'
+          go-version: 1.25.x
+      - name: Run Fuzzing
+        run: go test -fuzz=${{ matrix.fuzz_test }}$ -fuzztime=5m ./util/fuzzing
+        continue-on-error: true
+        id: fuzz
+      - name: Upload Crash Artifacts
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        if: failure()
         with:
-          name: artifacts
-          path: ./out/artifacts
+          name: fuzz-artifacts-${{ matrix.fuzz_test }}
+          path: util/fuzzing/testdata/fuzz/${{ matrix.fuzz_test }}
+  fuzzing_status:
+    # This status check aggregates the individual matrix jobs of the fuzzing
+    # step into a final status. Fails if a single matrix job fails, succeeds if
+    # all matrix jobs succeed.
+    name: Fuzzing
+    runs-on: ubuntu-latest
+    needs: [fuzzing]
+    if: always()
+    steps:
+      - name: Successful fuzzing
+        if: ${{ !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled')) }}
+        run: exit 0
+      - name: Failing or cancelled fuzzing
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+        run: exit 1

CHANGELOG.md

@@ -54,7 +54,7 @@
 * [FEATURE] OAuth2: support jwt-bearer grant-type (RFC7523 3.1). #17592
 * [FEATURE] Dockerfile: Add OpenContainers spec labels to Dockerfile. #16483
 * [FEATURE] SD: Add unified AWS service discovery for ec2, lightsail and ecs services. #17406
-* [FEATURE] Native histograms are now a stable, but optional feature, use the `scrape_native_histogram` config setting. #17232 #17315
+* [FEATURE] Native histograms are now a stable, but optional feature, use the `scrape_native_histograms` config setting. #17232 #17315
 * [FEATURE] UI: Support anchored and smoothed keyword in promql editor. #17239
 * [FEATURE] UI: Show detailed relabeling steps for each discovered target. #17337
 * [FEATURE] Alerting: Add urlQueryEscape to template functions. #17403

Dockerfile

@@ -9,7 +9,8 @@ LABEL org.opencontainers.image.authors="The Prometheus Authors" \
       org.opencontainers.image.source="https://github.com/prometheus/prometheus" \
       org.opencontainers.image.url="https://github.com/prometheus/prometheus" \
       org.opencontainers.image.documentation="https://prometheus.io/docs" \
-      org.opencontainers.image.licenses="Apache License 2.0"
+      org.opencontainers.image.licenses="Apache License 2.0" \
+      io.prometheus.image.variant="busybox"
 
 ARG ARCH="amd64"
 ARG OS="linux"

Dockerfile.distroless

@@ -0,0 +1,29 @@
+ARG DISTROLESS_ARCH="amd64"
+
+# Use DISTROLESS_ARCH for base image selection (handles armv7->arm mapping).
+FROM gcr.io/distroless/static-debian13:nonroot-${DISTROLESS_ARCH}
+# Base image sets USER to 65532:65532 (nonroot user).
+
+ARG ARCH="amd64"
+ARG OS="linux"
+
+LABEL org.opencontainers.image.authors="The Prometheus Authors"
+LABEL org.opencontainers.image.vendor="Prometheus"
+LABEL org.opencontainers.image.title="Prometheus"
+LABEL org.opencontainers.image.description="The Prometheus monitoring system and time series database"
+LABEL org.opencontainers.image.source="https://github.com/prometheus/prometheus"
+LABEL org.opencontainers.image.url="https://github.com/prometheus/prometheus"
+LABEL org.opencontainers.image.documentation="https://prometheus.io/docs"
+LABEL org.opencontainers.image.licenses="Apache License 2.0"
+LABEL io.prometheus.image.variant="distroless"
+
+COPY documentation/examples/prometheus.yml  /etc/prometheus/prometheus.yml
+COPY LICENSE NOTICE npm_licenses.tar.bz2 /
+COPY .build/${OS}-${ARCH}/prometheus        /bin/prometheus
+COPY .build/${OS}-${ARCH}/promtool          /bin/promtool
+
+WORKDIR /prometheus
+EXPOSE     9090
+ENTRYPOINT [ "/bin/prometheus" ]
+CMD        [ "--config.file=/etc/prometheus/prometheus.yml", \
+             "--storage.tsdb.path=/prometheus" ]

Makefile

@@ -220,3 +220,8 @@ check-node-version:
 bump-go-version:
 	@echo ">> bumping Go minor version"
 	@./scripts/bump_go_version.sh
+
+.PHONY: generate-fuzzing-seed-corpus
+generate-fuzzing-seed-corpus:
+	@echo ">> Generating fuzzing seed corpus"
+	@$(GO) generate -tags fuzzing ./util/fuzzing/corpus_gen

Makefile.common

@@ -82,11 +82,32 @@ endif
 PREFIX                  ?= $(shell pwd)
 BIN_DIR                 ?= $(shell pwd)
 DOCKER_IMAGE_TAG        ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
-DOCKERFILE_PATH         ?= ./Dockerfile
 DOCKERBUILD_CONTEXT     ?= ./
 DOCKER_REPO             ?= prom
 
+# Check if deprecated DOCKERFILE_PATH is set
+ifdef DOCKERFILE_PATH
+$(error DOCKERFILE_PATH is deprecated. Use DOCKERFILE_VARIANTS ?= $(DOCKERFILE_PATH) in the Makefile)
+endif
+
 DOCKER_ARCHS            ?= amd64
+DOCKERFILE_VARIANTS     ?= Dockerfile $(wildcard Dockerfile.*)
+
+# Function to extract variant from Dockerfile label.
+# Returns the variant name from io.prometheus.image.variant label, or "default" if not found.
+define dockerfile_variant
+$(strip $(or $(shell sed -n 's/.*io\.prometheus\.image\.variant="\([^"]*\)".*/\1/p' $(1)),default))
+endef
+
+# Check for duplicate variant names (including default for Dockerfiles without labels).
+DOCKERFILE_VARIANT_NAMES := $(foreach df,$(DOCKERFILE_VARIANTS),$(call dockerfile_variant,$(df)))
+DOCKERFILE_VARIANT_NAMES_SORTED := $(sort $(DOCKERFILE_VARIANT_NAMES))
+ifneq ($(words $(DOCKERFILE_VARIANT_NAMES)),$(words $(DOCKERFILE_VARIANT_NAMES_SORTED)))
+$(error Duplicate variant names found. Each Dockerfile must have a unique io.prometheus.image.variant label, and only one can be without a label (default))
+endif
+
+# Build variant:dockerfile pairs for shell iteration.
+DOCKERFILE_VARIANTS_WITH_NAMES := $(foreach df,$(DOCKERFILE_VARIANTS),$(call dockerfile_variant,$(df)):$(df))
 
 BUILD_DOCKER_ARCHS = $(addprefix common-docker-,$(DOCKER_ARCHS))
 PUBLISH_DOCKER_ARCHS = $(addprefix common-docker-publish-,$(DOCKER_ARCHS))
@@ -226,28 +247,110 @@ common-docker-repo-name:
 .PHONY: common-docker $(BUILD_DOCKER_ARCHS)
 common-docker: $(BUILD_DOCKER_ARCHS)
 $(BUILD_DOCKER_ARCHS): common-docker-%:
-	docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \
-		-f $(DOCKERFILE_PATH) \
-		--build-arg ARCH="$*" \
-		--build-arg OS="linux" \
-		$(DOCKERBUILD_CONTEXT)
+	@for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \
+		dockerfile=$${variant#*:}; \
+		variant_name=$${variant%%:*}; \
+		distroless_arch="$*"; \
+		if [ "$*" = "armv7" ]; then \
+			distroless_arch="arm"; \
+		fi; \
+		if [ "$$dockerfile" = "Dockerfile" ]; then \
+			echo "Building default variant ($$variant_name) for linux-$* using $$dockerfile"; \
+			docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \
+				-f $$dockerfile \
+				--build-arg ARCH="$*" \
+				--build-arg OS="linux" \
+				--build-arg DISTROLESS_ARCH="$$distroless_arch" \
+				$(DOCKERBUILD_CONTEXT); \
+			if [ "$$variant_name" != "default" ]; then \
+				echo "Tagging default variant with $$variant_name suffix"; \
+				docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \
+					"$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \
+			fi; \
+		else \
+			echo "Building $$variant_name variant for linux-$* using $$dockerfile"; \
+			docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" \
+				-f $$dockerfile \
+				--build-arg ARCH="$*" \
+				--build-arg OS="linux" \
+				--build-arg DISTROLESS_ARCH="$$distroless_arch" \
+				$(DOCKERBUILD_CONTEXT); \
+		fi; \
+	done
 
 .PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS)
 common-docker-publish: $(PUBLISH_DOCKER_ARCHS)
 $(PUBLISH_DOCKER_ARCHS): common-docker-publish-%:
-	docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)"
+	@for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \
+		dockerfile=$${variant#*:}; \
+		variant_name=$${variant%%:*}; \
+		if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \
+			echo "Pushing $$variant_name variant for linux-$*"; \
+			docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \
+		fi; \
+		if [ "$$dockerfile" = "Dockerfile" ]; then \
+			echo "Pushing default variant ($$variant_name) for linux-$*"; \
+			docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)"; \
+		fi; \
+		if [ "$(DOCKER_IMAGE_TAG)" = "latest" ]; then \
+			if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \
+				echo "Pushing $$variant_name variant version tags for linux-$*"; \
+				docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \
+			fi; \
+			if [ "$$dockerfile" = "Dockerfile" ]; then \
+				echo "Pushing default variant version tag for linux-$*"; \
+				docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"; \
+			fi; \
+		fi; \
+	done
 
 DOCKER_MAJOR_VERSION_TAG = $(firstword $(subst ., ,$(shell cat VERSION)))
 .PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS)
 common-docker-tag-latest: $(TAG_DOCKER_ARCHS)
 $(TAG_DOCKER_ARCHS): common-docker-tag-latest-%:
-	docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest"
-	docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"
+	@for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \
+		dockerfile=$${variant#*:}; \
+		variant_name=$${variant%%:*}; \
+		if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \
+			echo "Tagging $$variant_name variant for linux-$* as latest"; \
+			docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest-$$variant_name"; \
+			docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \
+		fi; \
+		if [ "$$dockerfile" = "Dockerfile" ]; then \
+			echo "Tagging default variant ($$variant_name) for linux-$* as latest"; \
+			docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest"; \
+			docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"; \
+		fi; \
+	done
 
 .PHONY: common-docker-manifest
 common-docker-manifest:
-	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG))
-	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)"
+	@for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \
+		dockerfile=$${variant#*:}; \
+		variant_name=$${variant%%:*}; \
+		if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \
+			echo "Creating manifest for $$variant_name variant"; \
+			DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name); \
+			DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \
+		fi; \
+		if [ "$$dockerfile" = "Dockerfile" ]; then \
+			echo "Creating default variant ($$variant_name) manifest"; \
+			DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG)); \
+			DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)"; \
+		fi; \
+		if [ "$(DOCKER_IMAGE_TAG)" = "latest" ]; then \
+			if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \
+				echo "Creating manifest for $$variant_name variant version tag"; \
+				DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name); \
+				DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \
+			fi; \
+			if [ "$$dockerfile" = "Dockerfile" ]; then \
+				echo "Creating default variant version tag manifest"; \
+				DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):v$(DOCKER_MAJOR_VERSION_TAG)); \
+				DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)"; \
+			fi; \
+		fi; \
+	done
 
 .PHONY: promu
 promu: $(PROMU)

RELEASE.md

@@ -7,19 +7,20 @@ This page describes the release process and the currently planned schedule for u
 Release cadence of first pre-releases being cut is 6 weeks.
 Please see [the v2.55 RELEASE.md](https://github.com/prometheus/prometheus/blob/release-2.55/RELEASE.md) for the v2 release series schedule.
 
-| release series | date of first pre-release (year-month-day) | release shepherd                   |
-|----------------|--------------------------------------------|------------------------------------|
-| v3.0           | 2024-11-14                                 | Jan Fajerski (GitHub: @jan--f)     |
-| v3.1           | 2024-12-17                                 | Bryan Boreham (GitHub: @bboreham)  |
-| v3.2           | 2025-01-28                                 | Jan Fajerski (GitHub: @jan--f)     |
-| v3.3           | 2025-03-11                                 | Ayoub Mrini (Github: @machine424)  |
-| v3.4           | 2025-04-29                                 | Jan-Otto Kröpke (Github: @jkroepke)|
-| v3.5 LTS       | 2025-06-03                                 | Bryan Boreham (GitHub: @bboreham)  |
-| v3.6           | 2025-08-01                                 | Ayoub Mrini (Github: @machine424)  |
-| v3.7           | 2025-09-25                                 | Arthur Sens and George Krajcsovits (Github: @ArthurSens and @krajorama)|
-| v3.8           | 2025-11-06                                 | Jan Fajerski (GitHub: @jan--f)     |
-| v3.9           | 2025-12-18                                 | Bryan Boreham (GitHub: @bboreham)  |
-| v3.10          | 2026-02-05                                 | **volunteer welcome**              |
+| release series | date of first pre-release (year-month-day) | release shepherd                                                        |
+|----------------|--------------------------------------------|-------------------------------------------------------------------------|
+| v3.0           | 2024-11-14                                 | Jan Fajerski (GitHub: @jan--f)                                          |
+| v3.1           | 2024-12-17                                 | Bryan Boreham (GitHub: @bboreham)                                       |
+| v3.2           | 2025-01-28                                 | Jan Fajerski (GitHub: @jan--f)                                          |
+| v3.3           | 2025-03-11                                 | Ayoub Mrini (Github: @machine424)                                       |
+| v3.4           | 2025-04-29                                 | Jan-Otto Kröpke (Github: @jkroepke)                                     |
+| v3.5 LTS       | 2025-06-03                                 | Bryan Boreham (GitHub: @bboreham)                                       |
+| v3.6           | 2025-08-01                                 | Ayoub Mrini (Github: @machine424)                                       |
+| v3.7           | 2025-09-25                                 | Arthur Sens and George Krajcsovits (Github: @ArthurSens and @krajorama) |
+| v3.8           | 2025-11-06                                 | Jan Fajerski (GitHub: @jan--f)                                          |
+| v3.9           | 2025-12-18                                 | Bryan Boreham (GitHub: @bboreham)                                       |
+| v3.10          | 2026-02-05                                 | Ganesh Vernekar (Github: @codesome)                                     |
+| v3.11          | 2026-03-19                                 | **volunteer welcome**                                                   |
 
 If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
 

cmd/prometheus/main.go

@@ -281,6 +281,9 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error {
 			case "promql-extended-range-selectors":
 				parser.EnableExtendedRangeSelectors = true
 				logger.Info("Experimental PromQL extended range selectors enabled.")
+			case "promql-binop-fill-modifiers":
+				parser.EnableBinopFillModifiers = true
+				logger.Info("Experimental PromQL binary operator fill modifiers enabled.")
 			case "":
 				continue
 			case "old-ui":
@@ -578,7 +581,7 @@ func main() {
 	a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
 		Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		Default("").StringsVar(&cfg.featureList)
 
 	a.Flag("agent", "Run Prometheus in 'Agent mode'.").BoolVar(&agentMode)
@@ -1573,7 +1576,7 @@ func reloadConfig(filename string, enableExemplarStorage bool, logger *slog.Logg
 			logger.Error("Failed to apply configuration", "err", err)
 			failed = true
 		}
-		timingsLogger = timingsLogger.With((rl.name), time.Since(rstart))
+		timingsLogger = timingsLogger.With(rl.name, time.Since(rstart))
 	}
 	if failed {
 		return fmt.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename)
@@ -1747,6 +1750,14 @@ func (s *readyStorage) Appender(ctx context.Context) storage.Appender {
 	return notReadyAppender{}
 }
 
+// AppenderV2 implements the Storage interface.
+func (s *readyStorage) AppenderV2(ctx context.Context) storage.AppenderV2 {
+	if x := s.get(); x != nil {
+		return x.AppenderV2(ctx)
+	}
+	return notReadyAppenderV2{}
+}
+
 type notReadyAppender struct{}
 
 // SetOptions does nothing in this appender implementation.
@@ -1780,6 +1791,15 @@ func (notReadyAppender) Commit() error { return tsdb.ErrNotReady }
 
 func (notReadyAppender) Rollback() error { return tsdb.ErrNotReady }
 
+type notReadyAppenderV2 struct{}
+
+func (notReadyAppenderV2) Append(storage.SeriesRef, labels.Labels, int64, int64, float64, *histogram.Histogram, *histogram.FloatHistogram, storage.AOptions) (storage.SeriesRef, error) {
+	return 0, tsdb.ErrNotReady
+}
+func (notReadyAppenderV2) Commit() error { return tsdb.ErrNotReady }
+
+func (notReadyAppenderV2) Rollback() error { return tsdb.ErrNotReady }
+
 // Close implements the Storage interface.
 func (s *readyStorage) Close() error {
 	if x := s.get(); x != nil {

cmd/prometheus/testdata/features.json

@@ -28,6 +28,9 @@
     "by": true,
     "delayed_name_removal": false,
     "duration_expr": false,
+    "fill": false,
+    "fill_left": false,
+    "fill_right": false,
     "group_left": true,
     "group_right": true,
     "ignoring": true,

cmd/promtool/backfill.go

@@ -27,7 +27,6 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/textparse"
 	"github.com/prometheus/prometheus/tsdb"
-	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 )
 
 func getMinAndMaxTimestamps(p textparse.Parser) (int64, int64, error) {
@@ -94,7 +93,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
 		return err
 	}
 	defer func() {
-		returnErr = tsdb_errors.NewMulti(returnErr, db.Close()).Err()
+		returnErr = errors.Join(returnErr, db.Close())
 	}()
 
 	var (
@@ -125,7 +124,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
 				return fmt.Errorf("block writer: %w", err)
 			}
 			defer func() {
-				err = tsdb_errors.NewMulti(err, w.Close()).Err()
+				err = errors.Join(err, w.Close())
 			}()
 
 			ctx := context.Background()

cmd/promtool/rules.go

@@ -15,6 +15,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"time"
@@ -28,7 +29,6 @@ import (
 	"github.com/prometheus/prometheus/rules"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
-	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 )
 
 const maxSamplesInMemory = 5000
@@ -143,7 +143,7 @@ func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName
 		var closed bool
 		defer func() {
 			if !closed {
-				err = tsdb_errors.NewMulti(err, w.Close()).Err()
+				err = errors.Join(err, w.Close())
 			}
 		}()
 		app := newMultipleAppender(ctx, w)
@@ -181,7 +181,7 @@ func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName
 		if err := app.flushAndCommit(ctx); err != nil {
 			return fmt.Errorf("flush and commit: %w", err)
 		}
-		err = tsdb_errors.NewMulti(err, w.Close()).Err()
+		err = errors.Join(err, w.Close())
 		closed = true
 	}
 

cmd/promtool/tsdb.go

@@ -43,7 +43,6 @@ import (
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/chunks"
-	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 	"github.com/prometheus/prometheus/tsdb/fileutil"
 	"github.com/prometheus/prometheus/tsdb/index"
 )
@@ -339,7 +338,7 @@ func listBlocks(path string, humanReadable bool) error {
 		return err
 	}
 	defer func() {
-		err = tsdb_errors.NewMulti(err, db.Close()).Err()
+		err = errors.Join(err, db.Close())
 	}()
 	blocks, err := db.Blocks()
 	if err != nil {
@@ -425,7 +424,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten
 		return err
 	}
 	defer func() {
-		err = tsdb_errors.NewMulti(err, db.Close()).Err()
+		err = errors.Join(err, db.Close())
 	}()
 
 	meta := block.Meta()
@@ -625,7 +624,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb.
 		return err
 	}
 	defer func() {
-		err = tsdb_errors.NewMulti(err, chunkr.Close()).Err()
+		err = errors.Join(err, chunkr.Close())
 	}()
 
 	totalChunks := 0
@@ -713,7 +712,7 @@ func dumpTSDBData(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt
 		return err
 	}
 	defer func() {
-		err = tsdb_errors.NewMulti(err, db.Close()).Err()
+		err = errors.Join(err, db.Close())
 	}()
 	q, err := db.Querier(mint, maxt)
 	if err != nil {
@@ -743,7 +742,7 @@ func dumpTSDBData(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt
 	}
 
 	if ws := ss.Warnings(); len(ws) > 0 {
-		return tsdb_errors.NewMulti(ws.AsErrors()...).Err()
+		return errors.Join(ws.AsErrors()...)
 	}
 
 	if ss.Err() != nil {

discovery/aws/aws.go

@@ -101,7 +101,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	switch c.Role {
 	case RoleEC2:
 		if c.EC2SDConfig == nil {
-			c.EC2SDConfig = &DefaultEC2SDConfig
+			ec2Config := DefaultEC2SDConfig
+			c.EC2SDConfig = &ec2Config
 		}
 		c.EC2SDConfig.HTTPClientConfig = c.HTTPClientConfig
 		if c.Region != "" {
@@ -133,7 +134,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error {
 		}
 	case RoleECS:
 		if c.ECSSDConfig == nil {
-			c.ECSSDConfig = &DefaultECSSDConfig
+			ecsConfig := DefaultECSSDConfig
+			c.ECSSDConfig = &ecsConfig
 		}
 		c.ECSSDConfig.HTTPClientConfig = c.HTTPClientConfig
 		if c.Region != "" {
@@ -165,7 +167,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error {
 		}
 	case RoleLightsail:
 		if c.LightsailSDConfig == nil {
-			c.LightsailSDConfig = &DefaultLightsailSDConfig
+			lightsailConfig := DefaultLightsailSDConfig
+			c.LightsailSDConfig = &lightsailConfig
 		}
 		c.LightsailSDConfig.HTTPClientConfig = c.HTTPClientConfig
 		if c.Region != "" {

discovery/aws/aws_test.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v3"
+	"go.yaml.in/yaml/v3"
 )
 
 func TestRoleUnmarshalYAML(t *testing.T) {
@@ -177,3 +177,109 @@ port: 9300`,
 		})
 	}
 }
+
+// TestMultipleSDConfigsDoNotShareState verifies that multiple AWS SD configs
+// don't share the same underlying configuration object. This was a bug where
+// all configs pointed to the same global default, causing port and other
+// settings from one job to overwrite settings in another job.
+func TestMultipleSDConfigsDoNotShareState(t *testing.T) {
+	tests := []struct {
+		name         string
+		yaml         string
+		validateFunc func(t *testing.T, cfg1, cfg2 *SDConfig)
+	}{
+		{
+			name: "EC2MultipleJobsDifferentPorts",
+			yaml: `
+- role: ec2
+  region: us-west-2
+  port: 9100
+  filters:
+    - name: tag:Name
+      values: [host-1]
+- role: ec2
+  region: us-west-2
+  port: 9101
+  filters:
+    - name: tag:Name
+      values: [host-2]`,
+			validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) {
+				require.Equal(t, RoleEC2, cfg1.Role)
+				require.Equal(t, RoleEC2, cfg2.Role)
+				require.NotNil(t, cfg1.EC2SDConfig)
+				require.NotNil(t, cfg2.EC2SDConfig)
+
+				// Verify ports are different and not shared
+				require.Equal(t, 9100, cfg1.EC2SDConfig.Port)
+				require.Equal(t, 9101, cfg2.EC2SDConfig.Port)
+
+				// Verify filters are different and not shared
+				require.Len(t, cfg1.EC2SDConfig.Filters, 1)
+				require.Len(t, cfg2.EC2SDConfig.Filters, 1)
+				require.Equal(t, []string{"host-1"}, cfg1.EC2SDConfig.Filters[0].Values)
+				require.Equal(t, []string{"host-2"}, cfg2.EC2SDConfig.Filters[0].Values)
+
+				// Most importantly: verify they're not the same pointer
+				require.NotSame(t, cfg1.EC2SDConfig, cfg2.EC2SDConfig,
+					"EC2SDConfig objects should not share the same memory address")
+			},
+		},
+		{
+			name: "ECSMultipleJobsDifferentPorts",
+			yaml: `
+- role: ecs
+  region: us-east-1
+  port: 8080
+  clusters: [cluster-a]
+- role: ecs
+  region: us-east-1
+  port: 8081
+  clusters: [cluster-b]`,
+			validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) {
+				require.Equal(t, RoleECS, cfg1.Role)
+				require.Equal(t, RoleECS, cfg2.Role)
+				require.NotNil(t, cfg1.ECSSDConfig)
+				require.NotNil(t, cfg2.ECSSDConfig)
+
+				require.Equal(t, 8080, cfg1.ECSSDConfig.Port)
+				require.Equal(t, 8081, cfg2.ECSSDConfig.Port)
+				require.Equal(t, []string{"cluster-a"}, cfg1.ECSSDConfig.Clusters)
+				require.Equal(t, []string{"cluster-b"}, cfg2.ECSSDConfig.Clusters)
+
+				require.NotSame(t, cfg1.ECSSDConfig, cfg2.ECSSDConfig,
+					"ECSSDConfig objects should not share the same memory address")
+			},
+		},
+		{
+			name: "LightsailMultipleJobsDifferentPorts",
+			yaml: `
+- role: lightsail
+  region: eu-west-1
+  port: 7070
+- role: lightsail
+  region: eu-west-1
+  port: 7071`,
+			validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) {
+				require.Equal(t, RoleLightsail, cfg1.Role)
+				require.Equal(t, RoleLightsail, cfg2.Role)
+				require.NotNil(t, cfg1.LightsailSDConfig)
+				require.NotNil(t, cfg2.LightsailSDConfig)
+
+				require.Equal(t, 7070, cfg1.LightsailSDConfig.Port)
+				require.Equal(t, 7071, cfg2.LightsailSDConfig.Port)
+
+				require.NotSame(t, cfg1.LightsailSDConfig, cfg2.LightsailSDConfig,
+					"LightsailSDConfig objects should not share the same memory address")
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var configs []SDConfig
+			require.NoError(t, yaml.Unmarshal([]byte(tt.yaml), &configs))
+			require.Len(t, configs, 2)
+			tt.validateFunc(t, &configs[0], &configs[1])
+		})
+	}
+}

docs/command-line/prometheus.md

@@ -59,7 +59,7 @@ The Prometheus monitoring server
 | <code class="text-nowrap">--query.timeout</code> | Maximum time a query may take before being aborted. Use with server mode only. | `2m` |
 | <code class="text-nowrap">--query.max-concurrency</code> | Maximum number of queries executed concurrently. Use with server mode only. | `20` |
 | <code class="text-nowrap">--query.max-samples</code> | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` |
-| <code class="text-nowrap">--enable-feature</code> <code class="text-nowrap">...<code class="text-nowrap"> | Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. |  |
+| <code class="text-nowrap">--enable-feature</code> <code class="text-nowrap">...<code class="text-nowrap"> | Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. |  |
 | <code class="text-nowrap">--agent</code> | Run Prometheus in 'Agent mode'. |  |
 | <code class="text-nowrap">--log.level</code> | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` |
 | <code class="text-nowrap">--log.format</code> | Output format of log messages. One of: [logfmt, json] | `logfmt` |

docs/feature_flags.md

@@ -67,12 +67,12 @@ Currently, Prometheus supports start timestamps on the
 
 * `PrometheusProto`
 * `OpenMetrics1.0.0`
-  
+
 
 From the above, Prometheus recommends `PrometheusProto`. This is because OpenMetrics 1.0 Start Timestamp information is shared as a `<metric>_created` metric and parsing those
 are prone to errors and expensive (thus, adding an overhead). You also need to be careful to not pollute your Prometheus with extra `_created` metrics.
-  
-Therefore, when `created-timestamp-zero-ingestion` is enabled Prometheus changes the global `scrape_protocols` default configuration option to 
+
+Therefore, when `created-timestamp-zero-ingestion` is enabled Prometheus changes the global `scrape_protocols` default configuration option to
 `[ PrometheusProto, OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText0.0.4 ]`, resulting in negotiating the Prometheus Protobuf protocol first (unless the `scrape_protocols` option is set to a different value explicitly).
 
 Besides enabling this feature in Prometheus, start timestamps need to be exposed by the application being scraped.
@@ -288,8 +288,8 @@ when wrong types are used on wrong functions, automatic renames, delta types and
 
 ### Behavior with metadata records
 
-When this feature is enabled and the metadata WAL records exists, in an unlikely situation when type or unit are different across those, 
-the Prometheus outputs intends to prefer the `__type__` and `__unit__` labels values. For example on Remote Write 2.0, 
+When this feature is enabled and the metadata WAL records exists, in an unlikely situation when type or unit are different across those,
+the Prometheus outputs intends to prefer the `__type__` and `__unit__` labels values. For example on Remote Write 2.0,
 if  the metadata record somehow (e.g. due to bug) says "counter", but `__type__="gauge"` the remote time series will be set to a gauge.
 
 ## Use Uncached IO
@@ -338,9 +338,25 @@ Example query:
 
 > **Note for alerting and recording rules:**
 > The `smoothed` modifier requires samples after the evaluation interval, so using it directly in alerting or recording rules will typically *under-estimate* the result, as future samples are not available at evaluation time.
-> To use `smoothed` safely in rules, you **must** apply a `query_offset` to the rule group (see [documentation](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#rule_group)) to ensure the calculation window is fully in the past and all needed samples are available.  
+> To use `smoothed` safely in rules, you **must** apply a `query_offset` to the rule group (see [documentation](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#rule_group)) to ensure the calculation window is fully in the past and all needed samples are available.
 > For critical alerting, set the offset to at least one scrape interval; for less critical or more resilient use cases, consider a larger offset (multiple scrape intervals) to tolerate missed scrapes.
 
 For more details, see the [design doc](https://github.com/prometheus/proposals/blob/main/proposals/2025-04-04_extended-range-selectors-semantics.md).
 
 **Note**: Extended Range Selectors are not supported for subqueries.
+
+## Binary operator fill modifiers
+
+`--enable-feature=promql-binop-fill-modifiers`
+
+Enables experimental `fill()`, `fill_left()`, and `fill_right()` modifiers for PromQL binary operators. These modifiers allow filling in missing matches on either side of a binary operation with a provided default sample value.
+
+Example query:
+
+```
+  rate(successful_requests[5m])
++ fill(0)
+  rate(failed_requests[5m])
+```
+
+See [the fill modifiers documentation](querying/operators.md#filling-in-missing-matches) for more details and examples.

docs/querying/operators.md

@@ -47,9 +47,9 @@ special values like `NaN`, `+Inf`, and `-Inf`.
 scalar that is the result of the operator applied to both scalar operands.
 
 **Between an instant vector and a scalar**, the operator is applied to the
-value of every data sample in the vector. 
+value of every data sample in the vector.
 
-If the data sample is a float, the operation is performed between that float and the scalar. 
+If the data sample is a float, the operation is performed between that float and the scalar.
 For example, if an instant vector of float samples is multiplied by 2,
 the result is another vector of float samples in which every sample value of
 the original vector is multiplied by 2.
@@ -81,8 +81,9 @@ following:
 **Between two instant vectors**, a binary arithmetic operator is applied to
 each entry in the LHS vector and its [matching element](#vector-matching) in
 the RHS vector. The result is propagated into the result vector with the
-grouping labels becoming the output label set. Entries for which no matching
-entry in the right-hand vector can be found are not part of the result.
+grouping labels becoming the output label set. By default, series for which
+no matching entry in the opposite vector can be found are not part of the
+result. This behavior can be adjusted using [fill modifiers](#filling-in-missing-matches).
 
 If two float samples are matched, the arithmetic operator is applied to the two
 input values.
@@ -97,7 +98,7 @@ If two histogram samples are matched, only `+` and `-` are valid operations,
 each adding or subtracting all matching bucket populations and the count and
 the sum of observations. All other operations result in the removal of the
 corresponding element from the output vector, flagged by an info-level
-annotation. The `+` and -` operations should generally only be applied to gauge
+annotation. The `+` and `-` operations should generally only be applied to gauge
 histograms, but PromQL allows them for counter histograms, too, to cover
 specific use cases, for which special attention is required to avoid problems
 with unaligned counter resets. (Certain incompatibilities of counter resets can
@@ -106,7 +107,7 @@ two counter histograms results in a counter histogram. All other combination of
 operands and all subtractions result in a gauge histogram.
 
 **In any arithmetic binary operation involving vectors**, the metric name is
-dropped. This occurs even if `__name__` is explicitly mentioned in `on` 
+dropped. This occurs even if `__name__` is explicitly mentioned in `on`
 (see https://github.com/prometheus/prometheus/issues/16631 for further discussion).
 
 **For any arithmetic binary operation that may result in a negative
@@ -156,9 +157,9 @@ info-level annotation.
 applied to matching entries. Vector elements for which the expression is not
 true or which do not find a match on the other side of the expression get
 dropped from the result, while the others are propagated into a result vector
-with the grouping labels becoming the output label set. 
+with the grouping labels becoming the output label set.
 
-Matches between two float samples work as usual. 
+Matches between two float samples work as usual.
 
 Matches between a float sample and a histogram sample are invalid, and the
 corresponding element is removed from the result vector, flagged by an info-level
@@ -171,8 +172,8 @@ comparison binary operations are again invalid.
 modifier changes the behavior in the following ways:
 
 * Vector elements which find a match on the other side of the expression but for
-  which the expression is false instead have the value `0` and vector elements
-  that do find a match and for which the expression is true have the value `1`. 
+  which the expression is false instead have the value `0`, and vector elements
+  that do find a match and for which the expression is true have the value `1`.
   (Note that elements with no match or invalid operations involving histogram
   samples still return no result rather than the value `0`.)
 * The metric name is dropped.
@@ -216,11 +217,10 @@ matching behavior: One-to-one and many-to-one/one-to-many.
 
 ### Vector matching keywords
 
-These vector matching keywords allow for matching between series with different label sets
-providing:
+These vector matching keywords allow for matching between series with different label sets:
 
-* `on`
-* `ignoring`
+* `on(<label list>)`: Only match on provided labels.
+* `ignoring(<label list>)`: Ignore provided labels when matching.
 
 Label lists provided to matching keywords will determine how vectors are combined. Examples
 can be found in [One-to-one vector matches](#one-to-one-vector-matches) and in
@@ -230,8 +230,8 @@ can be found in [One-to-one vector matches](#one-to-one-vector-matches) and in
 
 These group modifiers enable many-to-one/one-to-many vector matching:
 
-* `group_left`
-* `group_right`
+* `group_left`: Allow many-to-one matching, where the left vector has higher cardinality.
+* `group_right`: Allow one-to-many matching, where the right vector has higher cardinality.
 
 Label lists can be provided to the group modifier which contain labels from the "one"-side to
 be included in the result metrics.
@@ -239,11 +239,9 @@ be included in the result metrics.
 _Many-to-one and one-to-many matching are advanced use cases that should be carefully considered.
 Often a proper use of `ignoring(<labels>)` provides the desired outcome._
 
-_Grouping modifiers can only be used for
-[comparison](#comparison-binary-operators) and
-[arithmetic](#arithmetic-binary-operators). Operations as `and`, `unless` and
-`or` operations match with all possible entries in the right vector by
-default._
+_Grouping modifiers can only be used for [comparison](#comparison-binary-operators),
+[arithmetic](#arithmetic-binary-operators), and [trigonometric](#trigonometric-binary-operators)
+operators. Set operators match with all possible entries on either side by default._
 
 ### One-to-one vector matches
 
@@ -311,6 +309,58 @@ left:
     {method="post", code="500"} 0.05            //   6 / 120
     {method="post", code="404"} 0.175           //  21 / 120
 
+### Filling in missing matches
+
+Fill modifiers are **experimental** and must be enabled with `--enable-feature=promql-binop-fill-modifiers`.
+
+By default, vector elements that do not find a match on the other side of a binary operation
+are not included in the result vector. Fill modifiers allow overriding this behavior by filling
+in missing series on either side of a binary operation with a provided default sample value:
+
+* `fill(<value>)`: Fill in missing matches on either side with `value`.
+* `fill_left(<value>)`: Fill in missing matches on the left side with `value`.
+* `fill_right(<value>)`: Fill in missing matches on the right side with `value`.
+
+`value` has to be a numeric literal representing a float sample. Histogram samples are not supported.
+
+Note that these modifiers can only fill in series that are missing on one side of the operation.
+If a series is missing on both sides, it cannot be created by these modifiers.
+
+The fill modifiers can be used in the following combinations:
+
+* `fill(<default>)`
+* `fill_left(<default>)`
+* `fill_right(<default>)`
+* `fill_left(<default>) fill_right(<default>)`
+* `fill_right(<default>) fill_left(<default>)`
+
+If other binary operator modifiers like `bool`, `on`, `ignoring`, `group_left`, or `group_right`
+are used, the fill modifiers must be provided last.
+
+When using fill modifiers in combination with `group_left` or `group_right`, they behave as follows:
+
+* If a fill modifier is used on the "many" side of a match, it will only fill in a single series
+  for the "many" side of each match group, using the group's matching labels as the series identity.
+* If a fill modifier is used on the "one" side of a match and the grouping modifier specifies
+  label names to include from the "one" side (e.g. `left_vector * on(instance, job) group_left(info_label) fill_right(1) right_vector`), those labels will not be filled in for missing
+  series, as there is no source for their values.
+
+Fill modifiers are not supported for set operators (`and`, `or`, `unless`), as the purpose of those
+operators is to filter series based on presence or absence in the other vector.
+
+Example query, filling in missing series on the either side with `0`:
+
+    method_code:http_errors:rate5m{status="500"} / ignoring(code) fill(0) method:http_requests:rate5m
+
+This returns a result vector containing the fraction of HTTP requests with status code
+of 500 for each method, as measured over the last 5 minutes. The entries with methods `put` and `del`
+are now included in the result with a filled-in default sample value of `0`, as they had no matching
+series on the respective other side:
+
+    {method="get"}  0.04            #  24 / 600
+    {method="put"}  +Inf            #   3 /   0 (missing right side filled in)
+    {method="del"}  0               #   0 /  34 (missing left side filled in)
+    {method="post"} 0.05            #   6 / 120
 
 ## Aggregation operators
 
@@ -357,7 +407,7 @@ identical between all elements of the vector.
 #### `sum`
 
 `sum(v)` sums up sample values in `v` in the same way as the `+` binary operator does
-between two values. 
+between two values.
 
 All sample values being aggregated into a single resulting vector element must either be
 float samples or histogram samples. An aggregation of a mix of both is invalid,
@@ -393,7 +443,7 @@ vector, flagged by a warn-level annotation.
 
 #### `min` and `max`
 
-`min(v)` and `max(v)` return the minimum or maximum value, respectively, in `v`. 
+`min(v)` and `max(v)` return the minimum or maximum value, respectively, in `v`.
 
 They only operate on float samples, following IEEE 754 floating
 point arithmetic, which in particular implies that `NaN` is only ever
@@ -403,9 +453,9 @@ samples in the input vector are ignored, flagged by an info-level annotation.
 #### `topk` and `bottomk`
 
 `topk(k, v)` and `bottomk(k, v)` are different from other aggregators in that a subset of
-`k` values from the input samples, including the original labels, are returned in the result vector. 
+`k` values from the input samples, including the original labels, are returned in the result vector.
 
-`by` and `without` are only used to bucket the input vector. 
+`by` and `without` are only used to bucket the input vector.
 
 Similar to `min` and `max`, they only operate on float samples, considering `NaN` values
 to be farthest from the top or bottom, respectively. Histogram samples in the
@@ -415,7 +465,7 @@ If used in an instant query, `topk` and `bottomk` return series ordered by
 value in descending or ascending order, respectively. If used with `by` or
 `without`, then series within each bucket are sorted by value, and series in
 the same bucket are returned consecutively, but there is no guarantee that
-buckets of series will be returned in any particular order. 
+buckets of series will be returned in any particular order.
 
 No sorting applies to range queries.
 
@@ -428,11 +478,11 @@ To get the 5 instances with the highest memory consumption across all instances
 #### `limitk`
 
 `limitk(k, v)` returns a subset of `k` input samples, including
-the original labels in the result vector. 
+the original labels in the result vector.
 
 The subset is selected in a deterministic pseudo-random way.
-This happens independent of the sample type. 
-Therefore, it works for both float samples and histogram samples. 
+This happens independent of the sample type.
+Therefore, it works for both float samples and histogram samples.
 
 ##### Example
 
@@ -470,8 +520,8 @@ The value may be a float or histogram sample.
 
 #### `count_values`
 
-`count_values(l, v)` outputs one time series per unique sample value in `v`. 
-Each series has an additional label, given by `l`, and the label value is the 
+`count_values(l, v)` outputs one time series per unique sample value in `v`.
+Each series has an additional label, given by `l`, and the label value is the
 unique sample value. The value of each time series is the number of times that sample value was present.
 
 `count_values` works with both float samples and histogram samples. For the
@@ -486,7 +536,7 @@ To count the number of binaries running each build version we could write:
 
 #### `stddev`
 
-`stddev(v)` returns the standard deviation of `v`. 
+`stddev(v)` returns the standard deviation of `v`.
 
 `stddev` only works with float samples, following IEEE 754 floating
 point arithmetic. Histogram samples in the input vector are ignored, flagged by
@@ -494,7 +544,7 @@ an info-level annotation.
 
 #### `stdvar`
 
-`stdvar(v)` returns the standard deviation of `v`. 
+`stdvar(v)` returns the standard deviation of `v`.
 
 `stdvar` only works with float samples, following IEEE 754 floating
 point arithmetic. Histogram samples in the input vector are ignored, flagged by
@@ -510,12 +560,12 @@ are ignored, flagged by an info-level annotation.
 
 `NaN` is considered the smallest possible value.
 
-For example, `quantile(0.5, ...)` calculates the median, `quantile(0.95, ...)` the 95th percentile. 
+For example, `quantile(0.5, ...)` calculates the median, `quantile(0.95, ...)` the 95th percentile.
 
 Special cases:
 
 * For φ = `NaN`, `NaN` is returned.
-* For φ < 0, `-Inf` is returned. 
+* For φ < 0, `-Inf` is returned.
 * For φ > 1, `+Inf` is returned.
 
 ## Binary operator precedence

documentation/examples/remote_storage/go.mod

@@ -1,6 +1,6 @@
 module github.com/prometheus/prometheus/documentation/examples/remote_storage
 
-go 1.24.9
+go 1.24.0
 
 require (
 	github.com/alecthomas/kingpin/v2 v2.4.0

go.mod

@@ -1,6 +1,6 @@
 module github.com/prometheus/prometheus
 
-go 1.24.9
+go 1.24.0
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0
@@ -34,7 +34,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v1.0.0
 	github.com/google/go-cmp v0.7.0
-	github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f
+	github.com/google/pprof v0.0.0-20260111202518-71be6bfdd440
 	github.com/google/uuid v1.6.0
 	github.com/gophercloud/gophercloud/v2 v2.9.0
 	github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853
@@ -71,7 +71,6 @@ require (
 	go.opentelemetry.io/collector/consumer v1.48.0
 	go.opentelemetry.io/collector/pdata v1.48.0
 	go.opentelemetry.io/collector/processor v1.48.0
-	go.opentelemetry.io/collector/semconv v0.128.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.64.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0
 	go.opentelemetry.io/otel v1.39.0
@@ -84,8 +83,8 @@ require (
 	go.uber.org/atomic v1.11.0
 	go.uber.org/automaxprocs v1.6.0
 	go.uber.org/goleak v1.3.0
-	go.uber.org/multierr v1.11.0
 	go.yaml.in/yaml/v2 v2.4.3
+	go.yaml.in/yaml/v3 v3.0.4
 	golang.org/x/oauth2 v0.34.0
 	golang.org/x/sync v0.19.0
 	golang.org/x/sys v0.39.0
@@ -94,7 +93,6 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b
 	google.golang.org/grpc v1.78.0
 	google.golang.org/protobuf v1.36.11
-	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.34.3
 	k8s.io/apimachinery v0.34.3
 	k8s.io/client-go v0.34.3
@@ -115,7 +113,8 @@ require (
 	github.com/go-openapi/swag/stringutils v0.25.4 // indirect
 	github.com/go-openapi/swag/typeutils v0.25.4 // indirect
 	github.com/go-openapi/swag/yamlutils v0.25.4 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
 )
 

go.sum

@@ -236,8 +236,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f h1:HU1RgM6NALf/KW9HEY6zry3ADbDKcmpQ+hJedoNGQYQ=
-github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f/go.mod h1:67FPmZWbr+KDT/VlpWtw6sO9XSjpJmLuHpoLmWiTGgY=
+github.com/google/pprof v0.0.0-20260111202518-71be6bfdd440 h1:oKBqR+eQXiIM7X8K1JEg9aoTEePLq/c6Awe484abOuA=
+github.com/google/pprof v0.0.0-20260111202518-71be6bfdd440/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
 github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
 github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -571,8 +571,6 @@ go.opentelemetry.io/collector/processor/processortest v0.142.0 h1:wQnJeXDejBL6r8
 go.opentelemetry.io/collector/processor/processortest v0.142.0/go.mod h1:QU5SWj0L+92MSvQxZDjwWCsKssNDm+nD6SHn7IvviUE=
 go.opentelemetry.io/collector/processor/xprocessor v0.142.0 h1:7a1Crxrd5iBMVnebTxkcqxVkRHAlOBUUmNTUVUTnlCU=
 go.opentelemetry.io/collector/processor/xprocessor v0.142.0/go.mod h1:LY/GS2DiJILJKS3ynU3eOLLWSP8CmN1FtdpAMsVV8AU=
-go.opentelemetry.io/collector/semconv v0.128.0 h1:MzYOz7Vgb3Kf5D7b49pqqgeUhEmOCuT10bIXb/Cc+k4=
-go.opentelemetry.io/collector/semconv v0.128.0/go.mod h1:OPXer4l43X23cnjLXIZnRj/qQOjSuq4TgBLI76P9hns=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.64.0 h1:OXSUzgmIFkcC4An+mv+lqqZSndTffXpjAyoR+1f8k/A=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.64.0/go.mod h1:1A4GVLFIm54HFqVdOpWmukap7rgb0frrE3zWXohLPdM=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y=

go.work

@@ -1,4 +1,4 @@
-go 1.24.9
+go 1.24.0
 
 use (
 	.

internal/tools/go.mod

@@ -1,6 +1,6 @@
 module github.com/prometheus/prometheus/internal/tools
 
-go 1.24.9
+go 1.24.0
 
 require (
 	github.com/bufbuild/buf v1.62.1

model/labels/regexp.go

@@ -71,6 +71,10 @@ func NewFastRegexMatcher(v string) (*FastRegexMatcher, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		// Remove any capture operations before trying to optimize the remaining operations.
+		clearCapture(parsed)
+
 		if parsed.Op == syntax.OpConcat {
 			m.prefix, m.suffix, m.contains = optimizeConcatRegex(parsed)
 		}

model/labels/regexp_test.go

@@ -93,6 +93,7 @@ var (
 		"(.+)-(.+)-(.+)-(.+)-(.+)",
 		"((.*))(?i:f)((.*))o((.*))o((.*))",
 		"((.*))f((.*))(?i:o)((.*))o((.*))",
+		"(.*0.*)",
 	}
 	values = []string{
 		"foo", " foo bar", "bar", "buzz\nbar", "bar foo", "bfoo", "\n", "\nfoo", "foo\n", "hello foo world", "hello foo\n world", "",

model/rulefmt/rulefmt.go

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	"github.com/prometheus/common/model"
-	"gopkg.in/yaml.v3"
+	"go.yaml.in/yaml/v3"
 
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/promql"

model/rulefmt/rulefmt_test.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v3"
+	"go.yaml.in/yaml/v3"
 )
 
 func TestParseFileSuccess(t *testing.T) {

notifier/alertmanager_test.go

@@ -14,11 +14,15 @@
 package notifier
 
 import (
+	"log/slog"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
 )
 
 func TestPostPath(t *testing.T) {
@@ -60,3 +64,89 @@ func TestLabelSetNotReused(t *testing.T) {
 	// Target modified during alertmanager extraction
 	require.Equal(t, tg, makeInputTargetGroup())
 }
+
+// TestAlertmanagerSetSync verifies that sync properly manages sendloop lifecycle:
+// - Starts sendloops for new alertmanagers.
+// - Stops sendloops for removed alertmanagers.
+// - Does NOT stop sendloops that are still in use.
+// - Does NOT stop sendloops that were just created.
+func TestAlertmanagerSetSync(t *testing.T) {
+	reg := prometheus.NewRegistry()
+	alertmanagersDiscoveredFunc := func() float64 { return 0 }
+	metrics := newAlertMetrics(reg, alertmanagersDiscoveredFunc)
+	logger := slog.New(slog.DiscardHandler)
+	opts := &Options{QueueCapacity: 10, MaxBatchSize: DefaultMaxBatchSize}
+
+	cfg := config.DefaultAlertmanagerConfig
+
+	// Create alertmanagerSet
+	ams, err := newAlertmanagerSet(&cfg, opts, logger, metrics)
+	require.NoError(t, err)
+
+	defer func() {
+		ams.sync([]*targetgroup.Group{})
+		require.Empty(t, ams.sendLoops, "All sendloops should be cleaned up")
+	}()
+
+	// First sync: Add AM1 and AM2
+	tgs1 := []*targetgroup.Group{
+		{
+			Targets: []model.LabelSet{
+				{model.AddressLabel: "am1.example.com:9093"},
+				{model.AddressLabel: "am2.example.com:9093"},
+			},
+		},
+	}
+
+	ams.sync(tgs1)
+
+	require.Len(t, ams.sendLoops, 2, "AM1 and AM2 sendloops should be created")
+	require.Contains(t, ams.sendLoops, "http://am1.example.com:9093/api/v2/alerts", "AM1 sendloop should be created")
+	require.Contains(t, ams.sendLoops, "http://am2.example.com:9093/api/v2/alerts", "AM2 sendloop should be created")
+
+	am1Loop := ams.sendLoops["http://am1.example.com:9093/api/v2/alerts"]
+	am2Loop := ams.sendLoops["http://am2.example.com:9093/api/v2/alerts"]
+	require.NotNil(t, am1Loop)
+	require.NotNil(t, am2Loop)
+
+	// Second sync: Keep AM2, remove AM1, add AM3
+	tgs2 := []*targetgroup.Group{
+		{
+			Targets: []model.LabelSet{
+				{model.AddressLabel: "am2.example.com:9093"},
+				{model.AddressLabel: "am3.example.com:9093"},
+			},
+		},
+	}
+
+	ams.sync(tgs2)
+
+	require.Len(t, ams.sendLoops, 2)
+	require.NotContains(t, ams.sendLoops, "http://am1.example.com:9093/api/v2/alerts", "AM1 sendloop should be removed")
+	require.Contains(t, ams.sendLoops, "http://am2.example.com:9093/api/v2/alerts", "AM2 sendloop should be kept")
+	require.Contains(t, ams.sendLoops, "http://am3.example.com:9093/api/v2/alerts", "AM3 sendloop should be created")
+
+	am2LoopAfter := ams.sendLoops["http://am2.example.com:9093/api/v2/alerts"]
+	require.Same(t, am2Loop, am2LoopAfter, "AM2 sendloop should not be recreated")
+
+	am3Loop := ams.sendLoops["http://am3.example.com:9093/api/v2/alerts"]
+	require.NotNil(t, am3Loop, "AM3 sendloop should be created")
+
+	// Third sync: Keep only AM3, remove AM2
+	tgs3 := []*targetgroup.Group{
+		{
+			Targets: []model.LabelSet{
+				{model.AddressLabel: "am3.example.com:9093"},
+			},
+		},
+	}
+
+	ams.sync(tgs3)
+
+	require.Len(t, ams.sendLoops, 1)
+	require.NotContains(t, ams.sendLoops, "http://am2.example.com:9093/api/v2/alerts", "AM2 sendloop should be removed")
+	require.Contains(t, ams.sendLoops, "http://am3.example.com:9093/api/v2/alerts", "AM3 sendloop should be kept")
+
+	am3LoopAfter := ams.sendLoops["http://am3.example.com:9093/api/v2/alerts"]
+	require.Same(t, am3Loop, am3LoopAfter, "AM3 sendloop should not be recreated")
+}

notifier/alertmanagerset.go

@@ -16,6 +16,7 @@ package notifier
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"fmt"
 	"log/slog"
 	"net/http"
 	"sync"
@@ -26,6 +27,7 @@ import (
 
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
+	"github.com/prometheus/prometheus/model/labels"
 )
 
 // alertmanagerSet contains a set of Alertmanagers discovered via a group of service
@@ -33,16 +35,19 @@ import (
 type alertmanagerSet struct {
 	cfg    *config.AlertmanagerConfig
 	client *http.Client
+	opts   *Options
 
 	metrics *alertMetrics
 
 	mtx        sync.RWMutex
 	ams        []alertmanager
 	droppedAms []alertmanager
-	logger     *slog.Logger
+	sendLoops  map[string]*sendLoop
+
+	logger *slog.Logger
 }
 
-func newAlertmanagerSet(cfg *config.AlertmanagerConfig, logger *slog.Logger, metrics *alertMetrics) (*alertmanagerSet, error) {
+func newAlertmanagerSet(cfg *config.AlertmanagerConfig, opts *Options, logger *slog.Logger, metrics *alertMetrics) (*alertmanagerSet, error) {
 	client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, "alertmanager")
 	if err != nil {
 		return nil, err
@@ -59,10 +64,12 @@ func newAlertmanagerSet(cfg *config.AlertmanagerConfig, logger *slog.Logger, met
 	client.Transport = t
 
 	s := &alertmanagerSet{
-		client:  client,
-		cfg:     cfg,
-		logger:  logger,
-		metrics: metrics,
+		client:    client,
+		cfg:       cfg,
+		opts:      opts,
+		sendLoops: make(map[string]*sendLoop),
+		logger:    logger,
+		metrics:   metrics,
 	}
 	return s, nil
 }
@@ -86,36 +93,32 @@ func (s *alertmanagerSet) sync(tgs []*targetgroup.Group) {
 	s.mtx.Lock()
 	defer s.mtx.Unlock()
 	previousAms := s.ams
-	// Set new Alertmanagers and deduplicate them along their unique URL.
 	s.ams = []alertmanager{}
 	s.droppedAms = []alertmanager{}
 	s.droppedAms = append(s.droppedAms, allDroppedAms...)
-	seen := map[string]struct{}{}
 
+	// Deduplicate Alertmanagers and add sendloops for new Alertmanagers.
+	seen := map[string]struct{}{}
 	for _, am := range allAms {
 		us := am.url().String()
 		if _, ok := seen[us]; ok {
 			continue
 		}
 
-		// This will initialize the Counters for the AM to 0.
-		s.metrics.sent.WithLabelValues(us)
-		s.metrics.errors.WithLabelValues(us)
-
 		seen[us] = struct{}{}
 		s.ams = append(s.ams, am)
 	}
-	// Now remove counters for any removed Alertmanagers.
+	s.addSendLoops(s.ams)
+
+	// Populate a list of Alertmanagers to clean up,
+	// avoid cleaning up what we just added.
 	for _, am := range previousAms {
 		us := am.url().String()
 		if _, ok := seen[us]; ok {
 			continue
 		}
-		s.metrics.latencySummary.DeleteLabelValues(us)
-		s.metrics.latencyHistogram.DeleteLabelValues(us)
-		s.metrics.sent.DeleteLabelValues(us)
-		s.metrics.errors.DeleteLabelValues(us)
 		seen[us] = struct{}{}
+		s.cleanSendLoops(am)
 	}
 }
 
@@ -127,3 +130,62 @@ func (s *alertmanagerSet) configHash() (string, error) {
 	hash := md5.Sum(b)
 	return hex.EncodeToString(hash[:]), nil
 }
+
+func (s *alertmanagerSet) send(alerts ...*Alert) {
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
+
+	if len(s.cfg.AlertRelabelConfigs) > 0 {
+		alerts = relabelAlerts(s.cfg.AlertRelabelConfigs, labels.Labels{}, alerts)
+		if len(alerts) == 0 {
+			return
+		}
+	}
+
+	for _, sendLoop := range s.sendLoops {
+		sendLoop.add(alerts...)
+	}
+}
+
+// addSendLoops creates and starts a send loop for newly discovered alertmanager.
+// This function expects the caller to acquire needed locks.
+func (s *alertmanagerSet) addSendLoops(ams []alertmanager) {
+	for _, am := range ams {
+		us := am.url().String()
+		// Only add if sendloop doesn't already exist
+		if loop, exists := s.sendLoops[us]; exists {
+			loop.logger.Debug("Alertmanager already has send loop running, skipping")
+			continue
+		}
+		sendLoop := newSendLoop(us, s.client, s.cfg, s.opts, s.logger.With("alertmanager", us), s.metrics)
+		go sendLoop.loop()
+		s.sendLoops[us] = sendLoop
+	}
+}
+
+// cleanSendLoops stops and cleans the send loops for each removed alertmanager.
+// This function expects the caller to acquire needed locks.
+func (s *alertmanagerSet) cleanSendLoops(ams ...alertmanager) {
+	for _, am := range ams {
+		us := am.url().String()
+		if sendLoop, ok := s.sendLoops[us]; ok {
+			sendLoop.stop()
+			delete(s.sendLoops, us)
+		}
+	}
+}
+
+// startSendLoops starts a send loop for newly discovered alertmanager.
+// This function expects the caller to acquire needed locks.
+// This is mainly needed for testing where the loops are added as part of the test setup.
+func (s *alertmanagerSet) startSendLoops(ams []alertmanager) {
+	for _, am := range ams {
+		us := am.url().String()
+
+		if l, ok := s.sendLoops[us]; ok {
+			go l.loop()
+			continue
+		}
+		panic(fmt.Sprintf("send loop not found for %s", us))
+	}
+}

notifier/manager.go

@@ -14,16 +14,12 @@
 package notifier
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
-	"io"
 	"log/slog"
 	"net/http"
 	"net/url"
 	"sync"
-	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
@@ -55,13 +51,11 @@ var userAgent = version.PrometheusUserAgent()
 // Manager is responsible for dispatching alert notifications to an
 // alert manager service.
 type Manager struct {
-	queue []*Alert
-	opts  *Options
+	opts *Options
 
 	metrics *alertMetrics
 
-	more chan struct{}
-	mtx  sync.RWMutex
+	mtx sync.RWMutex
 
 	stopOnce      *sync.Once
 	stopRequested chan struct{}
@@ -114,23 +108,16 @@ func NewManager(o *Options, nameValidationScheme model.ValidationScheme, logger
 	}
 
 	n := &Manager{
-		queue:         make([]*Alert, 0, o.QueueCapacity),
-		more:          make(chan struct{}, 1),
 		stopRequested: make(chan struct{}),
 		stopOnce:      &sync.Once{},
 		opts:          o,
 		logger:        logger,
 	}
 
-	queueLenFunc := func() float64 { return float64(n.queueLen()) }
 	alertmanagersDiscoveredFunc := func() float64 { return float64(len(n.Alertmanagers())) }
 
-	n.metrics = newAlertMetrics(
-		o.Registerer,
-		o.QueueCapacity,
-		queueLenFunc,
-		alertmanagersDiscoveredFunc,
-	)
+	n.metrics = newAlertMetrics(o.Registerer, alertmanagersDiscoveredFunc)
+	n.metrics.queueCapacity.Set(float64(o.QueueCapacity))
 
 	return n
 }
@@ -163,7 +150,7 @@ func (n *Manager) ApplyConfig(conf *config.Config) error {
 	}
 
 	for k, cfg := range conf.AlertingConfig.AlertmanagerConfigs.ToMap() {
-		ams, err := newAlertmanagerSet(cfg, n.logger, n.metrics)
+		ams, err := newAlertmanagerSet(cfg, n.opts, n.logger, n.metrics)
 		if err != nil {
 			return err
 		}
@@ -176,86 +163,54 @@ func (n *Manager) ApplyConfig(conf *config.Config) error {
 		if oldAmSet, ok := configToAlertmanagers[hash]; ok {
 			ams.ams = oldAmSet.ams
 			ams.droppedAms = oldAmSet.droppedAms
+			// Only transfer sendLoops to the first new config with this hash.
+			// Subsequent configs with the same hash should not share the sendLoops
+			// map reference, as that would cause shared mutable state between
+			// alertmanagerSets (cleanup in one would affect the other).
+			oldAmSet.mtx.Lock()
+			if oldAmSet.sendLoops != nil {
+				ams.mtx.Lock()
+				ams.sendLoops = oldAmSet.sendLoops
+				oldAmSet.sendLoops = nil
+				ams.mtx.Unlock()
+			}
+			oldAmSet.mtx.Unlock()
 		}
 
 		amSets[k] = ams
 	}
 
+	// Clean up sendLoops that weren't transferred to new config.
+	// This happens when: (1) key was removed, or (2) key exists but hash changed.
+	// After the transfer loop above, any oldAmSet with non-nil sendLoops
+	// had its sendLoops NOT transferred (since we set it to nil on transfer).
+	for _, oldAmSet := range n.alertmanagers {
+		oldAmSet.mtx.Lock()
+		if oldAmSet.sendLoops != nil {
+			oldAmSet.cleanSendLoops(oldAmSet.ams...)
+		}
+		oldAmSet.mtx.Unlock()
+	}
+
 	n.alertmanagers = amSets
 
 	return nil
 }
 
-func (n *Manager) queueLen() int {
-	n.mtx.RLock()
-	defer n.mtx.RUnlock()
-
-	return len(n.queue)
-}
-
-func (n *Manager) nextBatch() []*Alert {
-	n.mtx.Lock()
-	defer n.mtx.Unlock()
-
-	var alerts []*Alert
-
-	if maxBatchSize := n.opts.MaxBatchSize; len(n.queue) > maxBatchSize {
-		alerts = append(make([]*Alert, 0, maxBatchSize), n.queue[:maxBatchSize]...)
-		n.queue = n.queue[maxBatchSize:]
-	} else {
-		alerts = append(make([]*Alert, 0, len(n.queue)), n.queue...)
-		n.queue = n.queue[:0]
-	}
-
-	return alerts
-}
-
 // Run dispatches notifications continuously, returning once Stop has been called and all
 // pending notifications have been drained from the queue (if draining is enabled).
 //
 // Dispatching of notifications occurs in parallel to processing target updates to avoid one starving the other.
 // Refer to https://github.com/prometheus/prometheus/issues/13676 for more details.
 func (n *Manager) Run(tsets <-chan map[string][]*targetgroup.Group) {
-	wg := sync.WaitGroup{}
-	wg.Add(2)
+	n.targetUpdateLoop(tsets)
 
-	go func() {
-		defer wg.Done()
-		n.targetUpdateLoop(tsets)
-	}()
-
-	go func() {
-		defer wg.Done()
-		n.sendLoop()
-		n.drainQueue()
-	}()
-
-	wg.Wait()
-	n.logger.Info("Notification manager stopped")
-}
-
-// sendLoop continuously consumes the notifications queue and sends alerts to
-// the configured Alertmanagers.
-func (n *Manager) sendLoop() {
-	for {
-		// If we've been asked to stop, that takes priority over sending any further notifications.
-		select {
-		case <-n.stopRequested:
-			return
-		default:
-			select {
-			case <-n.stopRequested:
-				return
-
-			case <-n.more:
-				n.sendOneBatch()
-
-				// If the queue still has items left, kick off the next iteration.
-				if n.queueLen() > 0 {
-					n.setMore()
-				}
-			}
-		}
+	n.mtx.Lock()
+	defer n.mtx.Unlock()
+	for _, ams := range n.alertmanagers {
+		ams.mtx.Lock()
+		ams.cleanSendLoops(ams.ams...)
+		ams.mtx.Unlock()
 	}
 }
 
@@ -280,33 +235,6 @@ func (n *Manager) targetUpdateLoop(tsets <-chan map[string][]*targetgroup.Group)
 	}
 }
 
-func (n *Manager) sendOneBatch() {
-	alerts := n.nextBatch()
-
-	if !n.sendAll(alerts...) {
-		n.metrics.dropped.Add(float64(len(alerts)))
-	}
-}
-
-func (n *Manager) drainQueue() {
-	if !n.opts.DrainOnShutdown {
-		if n.queueLen() > 0 {
-			n.logger.Warn("Draining remaining notifications on shutdown is disabled, and some notifications have been dropped", "count", n.queueLen())
-			n.metrics.dropped.Add(float64(n.queueLen()))
-		}
-
-		return
-	}
-
-	n.logger.Info("Draining any remaining notifications...")
-
-	for n.queueLen() > 0 {
-		n.sendOneBatch()
-	}
-
-	n.logger.Info("Remaining notifications drained")
-}
-
 func (n *Manager) reload(tgs map[string][]*targetgroup.Group) {
 	n.mtx.Lock()
 	defer n.mtx.Unlock()
@@ -324,44 +252,23 @@ func (n *Manager) reload(tgs map[string][]*targetgroup.Group) {
 // Send queues the given notification requests for processing.
 // Panics if called on a handler that is not running.
 func (n *Manager) Send(alerts ...*Alert) {
-	n.mtx.Lock()
-	defer n.mtx.Unlock()
+	// If we've been asked to stop, that takes priority over accepting new alerts.
+	select {
+	case <-n.stopRequested:
+		return
+	default:
+	}
+
+	n.mtx.RLock()
+	defer n.mtx.RUnlock()
 
 	alerts = relabelAlerts(n.opts.RelabelConfigs, n.opts.ExternalLabels, alerts)
 	if len(alerts) == 0 {
 		return
 	}
 
-	// Queue capacity should be significantly larger than a single alert
-	// batch could be.
-	if d := len(alerts) - n.opts.QueueCapacity; d > 0 {
-		alerts = alerts[d:]
-
-		n.logger.Warn("Alert batch larger than queue capacity, dropping alerts", "num_dropped", d)
-		n.metrics.dropped.Add(float64(d))
-	}
-
-	// If the queue is full, remove the oldest alerts in favor
-	// of newer ones.
-	if d := (len(n.queue) + len(alerts)) - n.opts.QueueCapacity; d > 0 {
-		n.queue = n.queue[d:]
-
-		n.logger.Warn("Alert notification queue full, dropping alerts", "num_dropped", d)
-		n.metrics.dropped.Add(float64(d))
-	}
-	n.queue = append(n.queue, alerts...)
-
-	// Notify sending goroutine that there are alerts to be processed.
-	n.setMore()
-}
-
-// setMore signals that the alert queue has items.
-func (n *Manager) setMore() {
-	// If we cannot send on the channel, it means the signal already exists
-	// and has not been consumed yet.
-	select {
-	case n.more <- struct{}{}:
-	default:
+	for _, ams := range n.alertmanagers {
+		ams.send(alerts...)
 	}
 }
 
@@ -403,158 +310,11 @@ func (n *Manager) DroppedAlertmanagers() []*url.URL {
 	return res
 }
 
-// sendAll sends the alerts to all configured Alertmanagers concurrently.
-// It returns true if the alerts could be sent successfully to at least one Alertmanager.
-func (n *Manager) sendAll(alerts ...*Alert) bool {
-	if len(alerts) == 0 {
-		return true
-	}
-
-	begin := time.Now()
-
-	// cachedPayload represent 'alerts' marshaled for Alertmanager API v2.
-	// Marshaling happens below. Reference here is for caching between
-	// for loop iterations.
-	var cachedPayload []byte
-
-	n.mtx.RLock()
-	amSets := n.alertmanagers
-	n.mtx.RUnlock()
-
-	var (
-		wg           sync.WaitGroup
-		amSetCovered sync.Map
-	)
-	for k, ams := range amSets {
-		var (
-			payload  []byte
-			err      error
-			amAlerts = alerts
-		)
-
-		ams.mtx.RLock()
-
-		if len(ams.ams) == 0 {
-			ams.mtx.RUnlock()
-			continue
-		}
-
-		if len(ams.cfg.AlertRelabelConfigs) > 0 {
-			amAlerts = relabelAlerts(ams.cfg.AlertRelabelConfigs, labels.Labels{}, alerts)
-			if len(amAlerts) == 0 {
-				ams.mtx.RUnlock()
-				continue
-			}
-			// We can't use the cached values from previous iteration.
-			cachedPayload = nil
-		}
-
-		switch ams.cfg.APIVersion {
-		case config.AlertmanagerAPIVersionV2:
-			{
-				if cachedPayload == nil {
-					openAPIAlerts := alertsToOpenAPIAlerts(amAlerts)
-
-					cachedPayload, err = json.Marshal(openAPIAlerts)
-					if err != nil {
-						n.logger.Error("Encoding alerts for Alertmanager API v2 failed", "err", err)
-						ams.mtx.RUnlock()
-						return false
-					}
-				}
-
-				payload = cachedPayload
-			}
-		default:
-			{
-				n.logger.Error(
-					fmt.Sprintf("Invalid Alertmanager API version '%v', expected one of '%v'", ams.cfg.APIVersion, config.SupportedAlertmanagerAPIVersions),
-					"err", err,
-				)
-				ams.mtx.RUnlock()
-				return false
-			}
-		}
-
-		if len(ams.cfg.AlertRelabelConfigs) > 0 {
-			// We can't use the cached values on the next iteration.
-			cachedPayload = nil
-		}
-
-		// Being here means len(ams.ams) > 0
-		amSetCovered.Store(k, false)
-		for _, am := range ams.ams {
-			wg.Add(1)
-
-			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(ams.cfg.Timeout))
-			defer cancel()
-
-			go func(ctx context.Context, k string, client *http.Client, url string, payload []byte, count int) {
-				err := n.sendOne(ctx, client, url, payload)
-				if err != nil {
-					n.logger.Error("Error sending alerts", "alertmanager", url, "count", count, "err", err)
-					n.metrics.errors.WithLabelValues(url).Add(float64(count))
-				} else {
-					amSetCovered.CompareAndSwap(k, false, true)
-				}
-
-				durationSeconds := time.Since(begin).Seconds()
-				n.metrics.latencySummary.WithLabelValues(url).Observe(durationSeconds)
-				n.metrics.latencyHistogram.WithLabelValues(url).Observe(durationSeconds)
-				n.metrics.sent.WithLabelValues(url).Add(float64(count))
-
-				wg.Done()
-			}(ctx, k, ams.client, am.url().String(), payload, len(amAlerts))
-		}
-
-		ams.mtx.RUnlock()
-	}
-
-	wg.Wait()
-
-	// Return false if there are any sets which were attempted (e.g. not filtered
-	// out) but have no successes.
-	allAmSetsCovered := true
-	amSetCovered.Range(func(_, value any) bool {
-		if !value.(bool) {
-			allAmSetsCovered = false
-			return false
-		}
-		return true
-	})
-
-	return allAmSetsCovered
-}
-
-func (n *Manager) sendOne(ctx context.Context, c *http.Client, url string, b []byte) error {
-	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(b))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("User-Agent", userAgent)
-	req.Header.Set("Content-Type", contentTypeJSON)
-	resp, err := n.opts.Do(ctx, c, req)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		io.Copy(io.Discard, resp.Body)
-		resp.Body.Close()
-	}()
-
-	// Any HTTP status 2xx is OK.
-	if resp.StatusCode/100 != 2 {
-		return fmt.Errorf("bad response status %s", resp.Status)
-	}
-
-	return nil
-}
-
 // Stop signals the notification manager to shut down and immediately returns.
 //
 // Run will return once the notification manager has successfully shut down.
 //
-// The manager will optionally drain any queued notifications before shutting down.
+// The manager will optionally drain send loops before shutting down.
 //
 // Stop is safe to call multiple times.
 func (n *Manager) Stop() {

notifier/metric.go

@@ -24,17 +24,13 @@ type alertMetrics struct {
 	latencyHistogram        *prometheus.HistogramVec
 	errors                  *prometheus.CounterVec
 	sent                    *prometheus.CounterVec
-	dropped                 prometheus.Counter
-	queueLength             prometheus.GaugeFunc
+	dropped                 *prometheus.CounterVec
+	queueLength             *prometheus.GaugeVec
 	queueCapacity           prometheus.Gauge
 	alertmanagersDiscovered prometheus.GaugeFunc
 }
 
-func newAlertMetrics(
-	r prometheus.Registerer,
-	queueCap int,
-	queueLen, alertmanagersDiscovered func() float64,
-) *alertMetrics {
+func newAlertMetrics(r prometheus.Registerer, alertmanagersDiscovered func() float64) *alertMetrics {
 	m := &alertMetrics{
 		latencySummary: prometheus.NewSummaryVec(prometheus.SummaryOpts{
 			Namespace:  namespace,
@@ -74,18 +70,18 @@ func newAlertMetrics(
 		},
 			[]string{alertmanagerLabel},
 		),
-		dropped: prometheus.NewCounter(prometheus.CounterOpts{
+		dropped: prometheus.NewCounterVec(prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "dropped_total",
 			Help:      "Total number of alerts dropped due to errors when sending to Alertmanager.",
-		}),
-		queueLength: prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		}, []string{alertmanagerLabel}),
+		queueLength: prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "queue_length",
 			Help:      "The number of alert notifications in the queue.",
-		}, queueLen),
+		}, []string{alertmanagerLabel}),
 		queueCapacity: prometheus.NewGauge(prometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -98,8 +94,6 @@ func newAlertMetrics(
 		}, alertmanagersDiscovered),
 	}
 
-	m.queueCapacity.Set(float64(queueCap))
-
 	if r != nil {
 		r.MustRegister(
 			m.latencySummary,

notifier/sendloop.go

@@ -0,0 +1,273 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package notifier
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/prometheus/prometheus/config"
+)
+
+type sendLoop struct {
+	alertmanagerURL string
+
+	cfg    *config.AlertmanagerConfig
+	client *http.Client
+	opts   *Options
+
+	metrics *alertMetrics
+
+	mtx      sync.RWMutex
+	queue    []*Alert
+	hasWork  chan struct{}
+	stopped  chan struct{}
+	stopOnce sync.Once
+
+	logger *slog.Logger
+}
+
+func newSendLoop(
+	alertmanagerURL string,
+	client *http.Client,
+	cfg *config.AlertmanagerConfig,
+	opts *Options,
+	logger *slog.Logger,
+	metrics *alertMetrics,
+) *sendLoop {
+	// This will initialize the Counters for the AM to 0 and set the static queue capacity gauge.
+	metrics.dropped.WithLabelValues(alertmanagerURL)
+	metrics.errors.WithLabelValues(alertmanagerURL)
+	metrics.sent.WithLabelValues(alertmanagerURL)
+	metrics.queueLength.WithLabelValues(alertmanagerURL)
+
+	return &sendLoop{
+		alertmanagerURL: alertmanagerURL,
+		client:          client,
+		cfg:             cfg,
+		opts:            opts,
+		logger:          logger,
+		metrics:         metrics,
+		queue:           make([]*Alert, 0, opts.QueueCapacity),
+		hasWork:         make(chan struct{}, 1),
+		stopped:         make(chan struct{}),
+	}
+}
+
+func (s *sendLoop) add(alerts ...*Alert) {
+	select {
+	case <-s.stopped:
+		return
+	default:
+	}
+
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
+
+	var dropped int
+	// Queue capacity should be significantly larger than a single alert
+	// batch could be.
+	if d := len(alerts) - s.opts.QueueCapacity; d > 0 {
+		s.logger.Warn("Alert batch larger than queue capacity, dropping alerts", "count", d)
+		dropped += d
+		alerts = alerts[d:]
+	}
+
+	// If the queue is full, remove the oldest alerts in favor
+	// of newer ones.
+	if d := (len(s.queue) + len(alerts)) - s.opts.QueueCapacity; d > 0 {
+		s.logger.Warn("Alert notification queue full, dropping alerts", "count", d)
+		dropped += d
+		s.queue = s.queue[d:]
+	}
+
+	s.queue = append(s.queue, alerts...)
+
+	// Notify sending goroutine that there are alerts to be processed.
+	// If we cannot send on the channel, it means the signal already exists
+	// and has not been consumed yet.
+	s.notifyWork()
+
+	s.metrics.queueLength.WithLabelValues(s.alertmanagerURL).Set(float64(len(s.queue)))
+	if dropped > 0 {
+		s.metrics.dropped.WithLabelValues(s.alertmanagerURL).Add(float64(dropped))
+	}
+}
+
+func (s *sendLoop) notifyWork() {
+	select {
+	case <-s.stopped:
+		return
+	case s.hasWork <- struct{}{}:
+	default:
+	}
+}
+
+func (s *sendLoop) stop() {
+	s.stopOnce.Do(func() {
+		s.logger.Debug("Stopping send loop")
+		close(s.stopped)
+
+		if s.opts.DrainOnShutdown {
+			s.drainQueue()
+		} else {
+			ql := s.queueLen()
+			s.logger.Warn("Alert notification queue not drained on shutdown, dropping alerts", "count", ql)
+			s.metrics.dropped.WithLabelValues(s.alertmanagerURL).Add(float64(ql))
+		}
+
+		s.metrics.latencySummary.DeleteLabelValues(s.alertmanagerURL)
+		s.metrics.latencyHistogram.DeleteLabelValues(s.alertmanagerURL)
+		s.metrics.sent.DeleteLabelValues(s.alertmanagerURL)
+		s.metrics.dropped.DeleteLabelValues(s.alertmanagerURL)
+		s.metrics.errors.DeleteLabelValues(s.alertmanagerURL)
+		s.metrics.queueLength.DeleteLabelValues(s.alertmanagerURL)
+	})
+}
+
+func (s *sendLoop) drainQueue() {
+	for s.queueLen() > 0 {
+		s.sendOneBatch()
+	}
+}
+
+func (s *sendLoop) queueLen() int {
+	s.mtx.RLock()
+	defer s.mtx.RUnlock()
+
+	return len(s.queue)
+}
+
+func (s *sendLoop) nextBatch() []*Alert {
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
+
+	var alerts []*Alert
+	if maxBatchSize := s.opts.MaxBatchSize; len(s.queue) > maxBatchSize {
+		alerts = append(make([]*Alert, 0, maxBatchSize), s.queue[:maxBatchSize]...)
+		s.queue = s.queue[maxBatchSize:]
+	} else {
+		alerts = append(make([]*Alert, 0, len(s.queue)), s.queue...)
+		s.queue = s.queue[:0]
+	}
+	s.metrics.queueLength.WithLabelValues(s.alertmanagerURL).Set(float64(len(s.queue)))
+
+	return alerts
+}
+
+func (s *sendLoop) sendOneBatch() {
+	alerts := s.nextBatch()
+
+	if !s.sendAll(alerts) {
+		s.metrics.dropped.WithLabelValues(s.alertmanagerURL).Add(float64(len(alerts)))
+	}
+}
+
+// loop continuously consumes the notifications queue and sends alerts to
+// the Alertmanager.
+func (s *sendLoop) loop() {
+	s.logger.Debug("Starting send loop")
+	for {
+		// If we've been asked to stop, that takes priority over sending any further notifications.
+		select {
+		case <-s.stopped:
+			return
+		default:
+			select {
+			case <-s.stopped:
+				return
+			case <-s.hasWork:
+				s.sendOneBatch()
+
+				// If the queue still has items left, kick off the next iteration.
+				if s.queueLen() > 0 {
+					s.notifyWork()
+				}
+			}
+		}
+	}
+}
+
+func (s *sendLoop) sendAll(alerts []*Alert) bool {
+	if len(alerts) == 0 {
+		return true
+	}
+
+	begin := time.Now()
+
+	var payload []byte
+	var err error
+	switch s.cfg.APIVersion {
+	case config.AlertmanagerAPIVersionV2:
+		openAPIAlerts := alertsToOpenAPIAlerts(alerts)
+		payload, err = json.Marshal(openAPIAlerts)
+		if err != nil {
+			s.logger.Error("Encoding alerts for Alertmanager API v2 failed", "err", err)
+			return false
+		}
+
+	default:
+		s.logger.Error(
+			fmt.Sprintf("Invalid Alertmanager API version '%v', expected one of '%v'", s.cfg.APIVersion, config.SupportedAlertmanagerAPIVersions),
+			"err", err,
+		)
+		return false
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(s.cfg.Timeout))
+	defer cancel()
+
+	if err := s.sendOne(ctx, s.client, s.alertmanagerURL, payload); err != nil {
+		s.logger.Error("Error sending alerts", "count", len(alerts), "err", err)
+		s.metrics.errors.WithLabelValues(s.alertmanagerURL).Add(float64(len(alerts)))
+		return false
+	}
+	durationSeconds := time.Since(begin).Seconds()
+	s.metrics.latencySummary.WithLabelValues(s.alertmanagerURL).Observe(durationSeconds)
+	s.metrics.latencyHistogram.WithLabelValues(s.alertmanagerURL).Observe(durationSeconds)
+	s.metrics.sent.WithLabelValues(s.alertmanagerURL).Add(float64(len(alerts)))
+
+	return true
+}
+
+func (s *sendLoop) sendOne(ctx context.Context, c *http.Client, url string, b []byte) error {
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(b))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", userAgent)
+	req.Header.Set("Content-Type", contentTypeJSON)
+	resp, err := s.opts.Do(ctx, c, req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		io.Copy(io.Discard, resp.Body)
+		resp.Body.Close()
+	}()
+
+	// Any HTTP status 2xx is OK.
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("bad response status %s", resp.Status)
+	}
+
+	return nil
+}

notifier/sendloop_test.go

@@ -0,0 +1,187 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package notifier
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"log/slog"
+	"net/http"
+	"strconv"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+func TestCustomDo(t *testing.T) {
+	const testURL = "http://testurl.com/"
+	const testBody = "testbody"
+
+	var received bool
+	h := sendLoop{
+		opts: &Options{
+			Do: func(_ context.Context, _ *http.Client, req *http.Request) (*http.Response, error) {
+				received = true
+				body, err := io.ReadAll(req.Body)
+
+				require.NoError(t, err)
+
+				require.Equal(t, testBody, string(body))
+
+				require.Equal(t, testURL, req.URL.String())
+
+				return &http.Response{
+					Body: io.NopCloser(bytes.NewBuffer(nil)),
+				}, nil
+			},
+		},
+	}
+
+	h.sendOne(context.Background(), nil, testURL, []byte(testBody))
+
+	require.True(t, received)
+}
+
+func TestHandlerNextBatch(t *testing.T) {
+	sendLoop := newSendLoop("http://mock", nil, &config.DefaultAlertmanagerConfig, &Options{MaxBatchSize: DefaultMaxBatchSize}, slog.New(slog.DiscardHandler), newAlertMetrics(prometheus.NewRegistry(), nil))
+
+	for i := range make([]struct{}, 2*DefaultMaxBatchSize+1) {
+		sendLoop.queue = append(sendLoop.queue, &Alert{
+			Labels: labels.FromStrings("alertname", strconv.Itoa(i)),
+		})
+	}
+	expected := append([]*Alert{}, sendLoop.queue...)
+
+	require.NoError(t, alertsEqual(expected[0:DefaultMaxBatchSize], sendLoop.nextBatch()))
+	require.NoError(t, alertsEqual(expected[DefaultMaxBatchSize:2*DefaultMaxBatchSize], sendLoop.nextBatch()))
+	require.NoError(t, alertsEqual(expected[2*DefaultMaxBatchSize:], sendLoop.nextBatch()))
+	require.Empty(t, sendLoop.queue)
+}
+
+func TestAddAlertsToQueue(t *testing.T) {
+	alert1 := &Alert{Labels: labels.FromStrings("alertname", "existing1")}
+	alert2 := &Alert{Labels: labels.FromStrings("alertname", "existing2")}
+
+	s := newSendLoop("http://foo.bar/", nil, nil, &Options{QueueCapacity: 5}, slog.New(slog.DiscardHandler), newAlertMetrics(prometheus.NewRegistry(), nil))
+	s.add(alert1, alert2)
+	require.Equal(t, []*Alert{alert1, alert2}, s.queue)
+	require.Len(t, s.queue, 2)
+
+	alert3 := &Alert{Labels: labels.FromStrings("alertname", "new1")}
+	alert4 := &Alert{Labels: labels.FromStrings("alertname", "new2")}
+
+	// Add new alerts to the queue, expect 0 dropped
+	s.add(alert3, alert4)
+	require.Zero(t, prom_testutil.ToFloat64(s.metrics.dropped.WithLabelValues(s.alertmanagerURL)))
+
+	// Verify all new alerts were added to the queue
+	require.Equal(t, []*Alert{alert1, alert2, alert3, alert4}, s.queue)
+	require.Len(t, s.queue, 4)
+}
+
+func TestAddAlertsToQueueExceedingCapacity(t *testing.T) {
+	alert1 := &Alert{Labels: labels.FromStrings("alertname", "alert1")}
+	alert2 := &Alert{Labels: labels.FromStrings("alertname", "alert2")}
+
+	s := newSendLoop("http://foo.bar/", nil, nil, &Options{QueueCapacity: 3}, slog.New(slog.DiscardHandler), newAlertMetrics(prometheus.NewRegistry(), nil))
+	s.add(alert1, alert2)
+
+	alert3 := &Alert{Labels: labels.FromStrings("alertname", "alert3")}
+	alert4 := &Alert{Labels: labels.FromStrings("alertname", "alert4")}
+
+	// Add new alerts to queue, expect 1 dropped
+	s.add(alert3, alert4)
+	require.Equal(t, 1.0, prom_testutil.ToFloat64(s.metrics.dropped.WithLabelValues(s.alertmanagerURL)))
+
+	// Verify all new alerts were added to the queue
+	require.Equal(t, []*Alert{alert2, alert3, alert4}, s.queue)
+}
+
+func TestAddAlertsToQueueExceedingTotalCapacity(t *testing.T) {
+	alert1 := &Alert{Labels: labels.FromStrings("alertname", "alert1")}
+	alert2 := &Alert{Labels: labels.FromStrings("alertname", "alert2")}
+
+	s := newSendLoop("http://foo.bar/", nil, nil, &Options{QueueCapacity: 3}, slog.New(slog.DiscardHandler), newAlertMetrics(prometheus.NewRegistry(), nil))
+	s.add(alert1, alert2)
+
+	alert3 := &Alert{Labels: labels.FromStrings("alertname", "alert3")}
+	alert4 := &Alert{Labels: labels.FromStrings("alertname", "alert4")}
+	alert5 := &Alert{Labels: labels.FromStrings("alertname", "alert5")}
+	alert6 := &Alert{Labels: labels.FromStrings("alertname", "alert6")}
+
+	// Add new alerts to queue, expect 3 dropped: 1 from new batch + 2 from existing queued items
+	s.add(alert3, alert4, alert5, alert6)
+	require.Equal(t, 3.0, prom_testutil.ToFloat64(s.metrics.dropped.WithLabelValues(s.alertmanagerURL)))
+
+	// Verify all new alerts were added to the queue
+	require.Equal(t, []*Alert{alert4, alert5, alert6}, s.queue)
+}
+
+func TestNextBatchAlertsFromQueue(t *testing.T) {
+	s := newSendLoop("http://foo.bar/", nil, nil, &Options{QueueCapacity: 5, MaxBatchSize: 3}, slog.New(slog.DiscardHandler), newAlertMetrics(prometheus.NewRegistry(), nil))
+
+	alert1 := &Alert{Labels: labels.FromStrings("alertname", "alert1")}
+	alert2 := &Alert{Labels: labels.FromStrings("alertname", "alert2")}
+	alert3 := &Alert{Labels: labels.FromStrings("alertname", "alert3")}
+	s.add(alert1, alert2, alert3)
+
+	// Test batch-size alerts in the queue
+	require.Equal(t, []*Alert{alert1, alert2, alert3}, s.nextBatch())
+	require.Empty(t, s.nextBatch())
+
+	// Test full queue
+	alert4 := &Alert{Labels: labels.FromStrings("alertname", "alert4")}
+	alert5 := &Alert{Labels: labels.FromStrings("alertname", "alert5")}
+	s.add(alert1, alert2, alert3, alert4, alert5)
+	require.Equal(t, []*Alert{alert1, alert2, alert3}, s.nextBatch())
+	require.Equal(t, []*Alert{alert4, alert5}, s.nextBatch())
+	require.Empty(t, s.nextBatch())
+}
+
+func TestMetrics(t *testing.T) {
+	const alertmanagerURL = "http://alertmanager:9093"
+
+	// Use a single registry throughout the test - this is critical to catch registry conflicts
+	reg := prometheus.NewRegistry()
+	alertmanagersDiscoveredFunc := func() float64 { return 0 }
+	metrics := newAlertMetrics(reg, alertmanagersDiscoveredFunc)
+
+	logger := slog.New(slog.DiscardHandler)
+	opts := &Options{QueueCapacity: 10, MaxBatchSize: DefaultMaxBatchSize}
+
+	// Create first sendLoop - this initializes metrics with the alertmanager URL label
+	sendLoop1 := newSendLoop(alertmanagerURL, nil, &config.DefaultAlertmanagerConfig, opts, logger, metrics)
+
+	// Verify metrics are initialized
+	require.Equal(t, 0.0, prom_testutil.ToFloat64(metrics.dropped.WithLabelValues(alertmanagerURL)))
+	require.Equal(t, 0.0, prom_testutil.ToFloat64(metrics.sent.WithLabelValues(alertmanagerURL)))
+
+	// Stop the sendLoop - this should clean up all metrics
+	sendLoop1.stop()
+
+	// Create second sendLoop with the same URL - this should NOT panic or conflict
+	// because metrics were properly cleaned up
+	sendLoop2 := newSendLoop(alertmanagerURL, nil, &config.DefaultAlertmanagerConfig, opts, logger, metrics)
+	defer sendLoop2.stop()
+
+	// Verify metrics are re-initialized correctly
+	require.Equal(t, 0.0, prom_testutil.ToFloat64(metrics.dropped.WithLabelValues(alertmanagerURL)))
+	require.Equal(t, 0.0, prom_testutil.ToFloat64(metrics.sent.WithLabelValues(alertmanagerURL)))
+}

notifier/util_test.go

@@ -15,6 +15,7 @@ package notifier
 
 import (
 	"testing"
+	"time"
 
 	"github.com/prometheus/alertmanager/api/v2/models"
 	"github.com/stretchr/testify/require"
@@ -25,3 +26,99 @@ import (
 func TestLabelsToOpenAPILabelSet(t *testing.T) {
 	require.Equal(t, models.LabelSet{"aaa": "111", "bbb": "222"}, labelsToOpenAPILabelSet(labels.FromStrings("aaa", "111", "bbb", "222")))
 }
+
+// Edge case tests for utility functions
+
+func TestLabelsToOpenAPILabelSetEmpty(t *testing.T) {
+	result := labelsToOpenAPILabelSet(labels.EmptyLabels())
+	require.Empty(t, result)
+}
+
+func TestLabelsToOpenAPILabelSetSpecialCharacters(t *testing.T) {
+	result := labelsToOpenAPILabelSet(labels.FromStrings(
+		"special/chars", "value with spaces",
+		"unicode", "αβγ",
+		"empty", "",
+	))
+
+	expected := models.LabelSet{
+		"special/chars": "value with spaces",
+		"unicode":       "αβγ",
+		"empty":         "",
+	}
+	require.Equal(t, expected, result)
+}
+
+func TestAlertsToOpenAPIAlertsEmpty(t *testing.T) {
+	result := alertsToOpenAPIAlerts([]*Alert{})
+	require.Empty(t, result)
+}
+
+func TestAlertsToOpenAPIAlertsNil(t *testing.T) {
+	result := alertsToOpenAPIAlerts(nil)
+	require.Empty(t, result)
+}
+
+func TestAlertsToOpenAPIAlertsSingle(t *testing.T) {
+	now := time.Now()
+	alert := &Alert{
+		Labels:       labels.FromStrings("alertname", "test", "severity", "critical"),
+		Annotations:  labels.FromStrings("summary", "Test alert"),
+		StartsAt:     now,
+		EndsAt:       now.Add(time.Hour),
+		GeneratorURL: "http://prometheus:9090/graph",
+	}
+
+	result := alertsToOpenAPIAlerts([]*Alert{alert})
+	require.Len(t, result, 1)
+
+	apiAlert := result[0]
+	require.Equal(t, "test", apiAlert.Labels["alertname"])
+	require.Equal(t, "critical", apiAlert.Labels["severity"])
+	require.Equal(t, "Test alert", apiAlert.Annotations["summary"])
+	require.Equal(t, "http://prometheus:9090/graph", string(apiAlert.GeneratorURL))
+}
+
+func TestAlertsToOpenAPIAlertsMultiple(t *testing.T) {
+	now := time.Now()
+	alerts := []*Alert{
+		{
+			Labels:      labels.FromStrings("alertname", "alert1"),
+			Annotations: labels.FromStrings("desc", "First alert"),
+			StartsAt:    now,
+			EndsAt:      now.Add(time.Hour),
+		},
+		{
+			Labels:      labels.FromStrings("alertname", "alert2"),
+			Annotations: labels.FromStrings("desc", "Second alert"),
+			StartsAt:    now.Add(time.Minute),
+			EndsAt:      now.Add(2 * time.Hour),
+		},
+	}
+
+	result := alertsToOpenAPIAlerts(alerts)
+	require.Len(t, result, 2)
+
+	require.Equal(t, "alert1", result[0].Labels["alertname"])
+	require.Equal(t, "alert2", result[1].Labels["alertname"])
+	require.Equal(t, "First alert", result[0].Annotations["desc"])
+	require.Equal(t, "Second alert", result[1].Annotations["desc"])
+}
+
+func TestAlertsToOpenAPIAlertsEmptyFields(t *testing.T) {
+	alert := &Alert{
+		Labels:       labels.EmptyLabels(),
+		Annotations:  labels.EmptyLabels(),
+		StartsAt:     time.Time{},
+		EndsAt:       time.Time{},
+		GeneratorURL: "",
+	}
+
+	result := alertsToOpenAPIAlerts([]*Alert{alert})
+	require.Len(t, result, 1)
+
+	apiAlert := result[0]
+	require.Empty(t, apiAlert.Labels)
+	require.Empty(t, apiAlert.Annotations)
+	require.Empty(t, string(apiAlert.GeneratorURL))
+}

promql/engine.go

@@ -2862,7 +2862,8 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 	if matching.Card == parser.CardManyToMany {
 		panic("many-to-many only allowed for set operators")
 	}
-	if len(lhs) == 0 || len(rhs) == 0 {
+	if (len(lhs) == 0 && len(rhs) == 0) ||
+		((len(lhs) == 0 || len(rhs) == 0) && matching.FillValues.RHS == nil && matching.FillValues.LHS == nil) {
 		return nil, nil // Short-circuit: nothing is going to match.
 	}
 
@@ -2910,17 +2911,9 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 	}
 	matchedSigs := enh.matchedSigs
 
-	// For all lhs samples find a respective rhs sample and perform
-	// the binary operation.
 	var lastErr error
-	for i, ls := range lhs {
-		sigOrd := lhsh[i].sigOrdinal
-
-		rs, found := rightSigs[sigOrd] // Look for a match in the rhs Vector.
-		if !found {
-			continue
-		}
 
+	doBinOp := func(ls, rs Sample, sigOrd int) {
 		// Account for potentially swapped sidedness.
 		fl, fr := ls.F, rs.F
 		hl, hr := ls.H, rs.H
@@ -2931,7 +2924,7 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 		floatValue, histogramValue, keep, info, err := vectorElemBinop(op, fl, fr, hl, hr, pos)
 		if err != nil {
 			lastErr = err
-			continue
+			return
 		}
 		if info != nil {
 			lastErr = info
@@ -2971,7 +2964,7 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 		}
 
 		if !keep && !returnBool {
-			continue
+			return
 		}
 
 		enh.Out = append(enh.Out, Sample{
@@ -2981,6 +2974,43 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 			DropName: returnBool,
 		})
 	}
+
+	// For all lhs samples, find a respective rhs sample and perform
+	// the binary operation.
+	for i, ls := range lhs {
+		sigOrd := lhsh[i].sigOrdinal
+
+		rs, found := rightSigs[sigOrd] // Look for a match in the rhs Vector.
+		if !found {
+			fill := matching.FillValues.RHS
+			if fill == nil {
+				continue
+			}
+			rs = Sample{
+				Metric: ls.Metric.MatchLabels(matching.On, matching.MatchingLabels...),
+				F:      *fill,
+			}
+		}
+
+		doBinOp(ls, rs, sigOrd)
+	}
+
+	// For any rhs samples which have not been matched, check if we need to
+	// perform the operation with a fill value from the lhs.
+	if fill := matching.FillValues.LHS; fill != nil {
+		for sigOrd, rs := range rightSigs {
+			if _, matched := matchedSigs[sigOrd]; matched {
+				continue // Already matched.
+			}
+			ls := Sample{
+				Metric: rs.Metric.MatchLabels(matching.On, matching.MatchingLabels...),
+				F:      *fill,
+			}
+
+			doBinOp(ls, rs, sigOrd)
+		}
+	}
+
 	return enh.Out, lastErr
 }
 
@@ -4418,9 +4448,9 @@ func extendFloats(floats []FPoint, mint, maxt int64, smoothed bool) []FPoint {
 		lastSampleIndex--
 	}
 
-	// TODO: Preallocate the length of the new list.
-	out := make([]FPoint, 0)
-	// Create the new floats list with the boundary samples and the inner samples.
+	count := max(lastSampleIndex-firstSampleIndex+1, 0)
+	out := make([]FPoint, 0, count+2)
+
 	out = append(out, FPoint{T: mint, F: left})
 	out = append(out, floats[firstSampleIndex:lastSampleIndex+1]...)
 	out = append(out, FPoint{T: maxt, F: right})

promql/engine_test.go

@@ -3747,12 +3747,12 @@ func TestHistogramRateWithFloatStaleness(t *testing.T) {
 		recoded bool
 	)
 
-	newc, recoded, app, err = app.AppendHistogram(nil, 0, h1.Copy(), false)
+	newc, recoded, app, err = app.AppendHistogram(nil, 0, 0, h1.Copy(), false)
 	require.NoError(t, err)
 	require.False(t, recoded)
 	require.Nil(t, newc)
 
-	newc, recoded, _, err = app.AppendHistogram(nil, 10, h1.Copy(), false)
+	newc, recoded, _, err = app.AppendHistogram(nil, 0, 10, h1.Copy(), false)
 	require.NoError(t, err)
 	require.False(t, recoded)
 	require.Nil(t, newc)
@@ -3762,7 +3762,7 @@ func TestHistogramRateWithFloatStaleness(t *testing.T) {
 	app, err = c2.Appender()
 	require.NoError(t, err)
 
-	app.Append(20, math.Float64frombits(value.StaleNaN))
+	app.Append(0, 20, math.Float64frombits(value.StaleNaN))
 
 	// Make a chunk with two normal histograms that have zero value.
 	h2 := histogram.Histogram{
@@ -3773,12 +3773,12 @@ func TestHistogramRateWithFloatStaleness(t *testing.T) {
 	app, err = c3.Appender()
 	require.NoError(t, err)
 
-	newc, recoded, app, err = app.AppendHistogram(nil, 30, h2.Copy(), false)
+	newc, recoded, app, err = app.AppendHistogram(nil, 0, 30, h2.Copy(), false)
 	require.NoError(t, err)
 	require.False(t, recoded)
 	require.Nil(t, newc)
 
-	newc, recoded, _, err = app.AppendHistogram(nil, 40, h2.Copy(), false)
+	newc, recoded, _, err = app.AppendHistogram(nil, 0, 40, h2.Copy(), false)
 	require.NoError(t, err)
 	require.False(t, recoded)
 	require.Nil(t, newc)

promql/histogram_stats_iterator_test.go

@@ -235,4 +235,6 @@ func (h *histogramIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64,
 
 func (*histogramIterator) AtT() int64 { return 0 }
 
+func (*histogramIterator) AtST() int64 { return 0 }
+
 func (*histogramIterator) Err() error { return nil }

promql/info.go

@@ -143,6 +143,23 @@ func (ev *evaluator) fetchInfoSeries(ctx context.Context, mat Matrix, ignoreSeri
 		}
 	}
 	if len(idLblValues) == 0 {
+		// Even when returning early, we need to remove __name__ from dataLabelMatchers
+		// since it's not a data label selector (it's used to select which info metrics
+		// to consider). Without this, combineWithInfoVector would incorrectly exclude
+		// series when only __name__ is specified in the selector.
+		for name, ms := range dataLabelMatchers {
+			for i, m := range ms {
+				if m.Name == labels.MetricName {
+					ms = slices.Delete(ms, i, i+1)
+					break
+				}
+			}
+			if len(ms) > 0 {
+				dataLabelMatchers[name] = ms
+			} else {
+				delete(dataLabelMatchers, name)
+			}
+		}
 		return nil, nil, nil
 	}
 
@@ -424,9 +441,10 @@ func (ev *evaluator) combineWithInfoVector(base, info Vector, ignoreSeries map[u
 		}
 
 		infoLbls := enh.lb.Labels()
-		if infoLbls.Len() == 0 {
-			// If there's at least one data label matcher not matching the empty string,
-			// we have to ignore this series as there are no matching info series.
+		if len(seenInfoMetrics) == 0 {
+			// No info series matched this base series. If there's at least one data
+			// label matcher not matching the empty string, we have to ignore this
+			// series as there are no matching info series.
 			allMatchersMatchEmpty := true
 			for _, ms := range dataLabelMatchers {
 				for _, m := range ms {

promql/parser/ast.go

@@ -318,6 +318,19 @@ type VectorMatching struct {
 	// Include contains additional labels that should be included in
 	// the result from the side with the lower cardinality.
 	Include []string
+	// Fill-in values to use when a series from one side does not find a match on the other side.
+	FillValues VectorMatchFillValues
+}
+
+// VectorMatchFillValues contains the fill values to use for Vector matching
+// when one side does not find a match on the other side.
+// When a fill value is nil, no fill is applied for that side, and there
+// is no output for the match group if there is no match.
+type VectorMatchFillValues struct {
+	// RHS is the fill value to use for the right-hand side.
+	RHS *float64
+	// LHS is the fill value to use for the left-hand side.
+	LHS *float64
 }
 
 // Visitor allows visiting a Node and its child nodes. The Visit method is

promql/parser/features.go

@@ -26,6 +26,8 @@ func RegisterFeatures(r features.Collector) {
 			switch keyword {
 			case "anchored", "smoothed":
 				r.Set(features.PromQL, keyword, EnableExtendedRangeSelectors)
+			case "fill", "fill_left", "fill_right":
+				r.Set(features.PromQL, keyword, EnableBinopFillModifiers)
 			default:
 				r.Enable(features.PromQL, keyword)
 			}

promql/parser/generated_parser.y

@@ -139,6 +139,9 @@ BOOL
 BY
 GROUP_LEFT
 GROUP_RIGHT
+FILL
+FILL_LEFT
+FILL_RIGHT
 IGNORING
 OFFSET
 SMOOTHED
@@ -190,7 +193,7 @@ START_METRIC_SELECTOR
 %type <int> int
 %type <uint> uint
 %type <float> number series_value signed_number signed_or_unsigned_number
-%type <node> step_invariant_expr aggregate_expr aggregate_modifier bin_modifier binary_expr bool_modifier expr function_call function_call_args function_call_body group_modifiers label_matchers matrix_selector number_duration_literal offset_expr anchored_expr smoothed_expr on_or_ignoring paren_expr string_literal subquery_expr unary_expr vector_selector duration_expr paren_duration_expr positive_duration_expr offset_duration_expr
+%type <node> step_invariant_expr aggregate_expr aggregate_modifier bin_modifier fill_modifiers binary_expr bool_modifier expr function_call function_call_args function_call_body group_modifiers fill_value label_matchers matrix_selector number_duration_literal offset_expr anchored_expr smoothed_expr on_or_ignoring paren_expr string_literal subquery_expr unary_expr vector_selector duration_expr paren_duration_expr positive_duration_expr offset_duration_expr
 
 %start start
 
@@ -302,7 +305,7 @@ binary_expr     : expr ADD     bin_modifier expr { $$ = yylex.(*parser).newBinar
 
 // Using left recursion for the modifier rules, helps to keep the parser stack small and
 // reduces allocations.
-bin_modifier    : group_modifiers;
+bin_modifier    : fill_modifiers;
 
 bool_modifier   : /* empty */
                         { $$ = &BinaryExpr{
@@ -346,6 +349,47 @@ group_modifiers: bool_modifier /* empty */
                         }
                 ;
 
+fill_modifiers: group_modifiers /* empty */
+                /* Only fill() */
+                | group_modifiers FILL fill_value
+                        {
+                        $$ = $1
+                        fill := $3.(*NumberLiteral).Val
+                        $$.(*BinaryExpr).VectorMatching.FillValues.LHS = &fill
+                        $$.(*BinaryExpr).VectorMatching.FillValues.RHS = &fill
+                        }
+                /* Only fill_left() */
+                | group_modifiers FILL_LEFT fill_value
+                        {
+                        $$ = $1
+                        fill := $3.(*NumberLiteral).Val
+                        $$.(*BinaryExpr).VectorMatching.FillValues.LHS = &fill
+                        }
+                /* Only fill_right() */
+                | group_modifiers FILL_RIGHT fill_value
+                        {
+                        $$ = $1
+                        fill := $3.(*NumberLiteral).Val
+                        $$.(*BinaryExpr).VectorMatching.FillValues.RHS = &fill
+                        }
+                /* fill_left() fill_right() */
+                | group_modifiers FILL_LEFT fill_value FILL_RIGHT fill_value
+                        {
+                        $$ = $1
+                        fill_left := $3.(*NumberLiteral).Val
+                        fill_right := $5.(*NumberLiteral).Val
+                        $$.(*BinaryExpr).VectorMatching.FillValues.LHS = &fill_left
+                        $$.(*BinaryExpr).VectorMatching.FillValues.RHS = &fill_right
+                        }
+                /* fill_right() fill_left() */
+                | group_modifiers FILL_RIGHT fill_value FILL_LEFT fill_value
+                        {
+                        fill_right := $3.(*NumberLiteral).Val
+                        fill_left := $5.(*NumberLiteral).Val
+                        $$.(*BinaryExpr).VectorMatching.FillValues.LHS = &fill_left
+                        $$.(*BinaryExpr).VectorMatching.FillValues.RHS = &fill_right
+                        }
+                ;
 
 grouping_labels : LEFT_PAREN grouping_label_list RIGHT_PAREN
                         { $$ = $2 }
@@ -387,6 +431,21 @@ grouping_label  : maybe_label
                         { yylex.(*parser).unexpected("grouping opts", "label"); $$ = Item{} }
                 ;
 
+fill_value      : LEFT_PAREN number_duration_literal RIGHT_PAREN
+                        {
+                        $$ = $2.(*NumberLiteral)
+                        }
+                | LEFT_PAREN unary_op number_duration_literal RIGHT_PAREN
+                        {
+                        nl := $3.(*NumberLiteral)
+                        if $2.Typ == SUB {
+                          nl.Val *= -1
+                        }
+                        nl.PosRange.Start = $2.Pos
+                        $$ = nl
+                        }
+                ;
+
 /*
  * Function calls.
  */
@@ -697,7 +756,7 @@ metric          : metric_identifier label_set
                 ;
 
 
-metric_identifier: AVG | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | IDENTIFIER |  LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | QUANTILE | STDDEV | STDVAR | SUM | TOPK | WITHOUT | START | END | LIMITK | LIMIT_RATIO | STEP | RANGE | ANCHORED | SMOOTHED;
+metric_identifier: AVG | BOTTOMK | BY | COUNT | COUNT_VALUES | FILL | FILL_LEFT | FILL_RIGHT | GROUP | IDENTIFIER |  LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | QUANTILE | STDDEV | STDVAR | SUM | TOPK | WITHOUT | START | END | LIMITK | LIMIT_RATIO | STEP | RANGE | ANCHORED | SMOOTHED;
 
 label_set       : LEFT_BRACE label_set_list RIGHT_BRACE
                         { $$ = labels.New($2...) }
@@ -791,14 +850,15 @@ series_item     : BLANK
                 // Histogram descriptions (part of unit testing).
                 | histogram_series_value
                         {
-                        $$ = []SequenceValue{{Histogram:$1}}
+                        $$ = []SequenceValue{yylex.(*parser).newHistogramSequenceValue($1)}
                         }
                 | histogram_series_value TIMES uint
                         {
                         $$ = []SequenceValue{}
                         // Add an additional value for time 0, which we ignore in tests.
+                        sv := yylex.(*parser).newHistogramSequenceValue($1)
                         for i:=uint64(0); i <= $3; i++{
-                                $$ = append($$, SequenceValue{Histogram:$1})
+                                $$ = append($$, sv)
                                 //$1 += $2
                         }
                         }
@@ -954,7 +1014,7 @@ counter_reset_hint : UNKNOWN_COUNTER_RESET | COUNTER_RESET | NOT_COUNTER_RESET |
 aggregate_op    : AVG | BOTTOMK | COUNT | COUNT_VALUES | GROUP | MAX | MIN | QUANTILE | STDDEV | STDVAR | SUM | TOPK | LIMITK | LIMIT_RATIO;
 
 // Inside of grouping options label names can be recognized as keywords by the lexer. This is a list of keywords that could also be a label name.
-maybe_label     : AVG | BOOL | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | GROUP_LEFT | GROUP_RIGHT | IDENTIFIER | IGNORING | LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | ON | QUANTILE | STDDEV | STDVAR | SUM | TOPK | START | END | ATAN2 | LIMITK | LIMIT_RATIO | STEP | RANGE | ANCHORED | SMOOTHED;
+maybe_label     : AVG | BOOL | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | GROUP_LEFT | GROUP_RIGHT | FILL | FILL_LEFT | FILL_RIGHT | IDENTIFIER | IGNORING | LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | ON | QUANTILE | STDDEV | STDVAR | SUM | TOPK | START | END | ATAN2 | LIMITK | LIMIT_RATIO | STEP | RANGE | ANCHORED | SMOOTHED;
 
 unary_op        : ADD | SUB;
 
@@ -1162,7 +1222,7 @@ offset_duration_expr    : number_duration_literal
                                 }
                         | duration_expr
                         ;
-                        
+
 min_max: MIN | MAX ;
 
 duration_expr   : number_duration_literal
@@ -1277,14 +1337,14 @@ duration_expr   : number_duration_literal
                 ;
 
 paren_duration_expr : LEFT_PAREN duration_expr RIGHT_PAREN
-                        { 
+                        {
                             yylex.(*parser).experimentalDurationExpr($2.(Expr))
                             if durationExpr, ok := $2.(*DurationExpr); ok {
                                 durationExpr.Wrapped = true
                                 $$ = durationExpr
                                 break
                             }
-                            $$ = $2 
+                            $$ = $2
                         }
                 ;
 

promql/parser/lex.go

@@ -137,6 +137,9 @@ var key = map[string]ItemType{
 	"ignoring":    IGNORING,
 	"group_left":  GROUP_LEFT,
 	"group_right": GROUP_RIGHT,
+	"fill":        FILL,
+	"fill_left":   FILL_LEFT,
+	"fill_right":  FILL_RIGHT,
 	"bool":        BOOL,
 
 	// Preprocessors.
@@ -1083,6 +1086,17 @@ Loop:
 			word := l.input[l.start:l.pos]
 			switch kw, ok := key[strings.ToLower(word)]; {
 			case ok:
+				// For fill/fill_left/fill_right, only treat as keyword if followed by '('
+				// This allows using these as metric names (e.g., "fill + fill").
+				// This could be done for other keywords as well, but for the new fill
+				// modifiers this is especially important so we don't break any existing
+				// queries.
+				if kw == FILL || kw == FILL_LEFT || kw == FILL_RIGHT {
+					if !l.peekFollowedByLeftParen() {
+						l.emit(IDENTIFIER)
+						break Loop
+					}
+				}
 				l.emit(kw)
 			case !strings.Contains(word, ":"):
 				l.emit(IDENTIFIER)
@@ -1098,6 +1112,23 @@ Loop:
 	return lexStatements
 }
 
+// peekFollowedByLeftParen checks if the next non-whitespace character is '('.
+// This is used for context-sensitive keywords like fill/fill_left/fill_right
+// that should only be treated as keywords when followed by '('.
+func (l *Lexer) peekFollowedByLeftParen() bool {
+	pos := l.pos
+	for {
+		if int(pos) >= len(l.input) {
+			return false
+		}
+		r, w := utf8.DecodeRuneInString(l.input[pos:])
+		if !isSpace(r) {
+			return r == '('
+		}
+		pos += posrange.Pos(w)
+	}
+}
+
 func isSpace(r rune) bool {
 	return r == ' ' || r == '\t' || r == '\n' || r == '\r'
 }

promql/parser/parse.go

@@ -45,6 +45,9 @@ var ExperimentalDurationExpr bool
 // EnableExtendedRangeSelectors is a flag to enable experimental extended range selectors.
 var EnableExtendedRangeSelectors bool
 
+// EnableBinopFillModifiers is a flag to enable experimental fill modifiers for binary operators.
+var EnableBinopFillModifiers bool
+
 type Parser interface {
 	ParseExpr() (Expr, error)
 	Close()
@@ -67,6 +70,11 @@ type parser struct {
 
 	generatedParserResult any
 	parseErrors           ParseErrors
+
+	// lastHistogramCounterResetHintSet is set to true when the most recently
+	// built histogram had a counter_reset_hint explicitly specified.
+	// This is used to populate CounterResetHintSet in SequenceValue.
+	lastHistogramCounterResetHintSet bool
 }
 
 type Opt func(p *parser)
@@ -234,6 +242,11 @@ type SequenceValue struct {
 	Value     float64
 	Omitted   bool
 	Histogram *histogram.FloatHistogram
+	// CounterResetHintSet is true if the counter reset hint was explicitly
+	// specified in the test file using counter_reset_hint:... syntax.
+	// This allows distinguishing between "no hint specified" (don't care)
+	// vs "counter_reset_hint:unknown" (verify it's unknown).
+	CounterResetHintSet bool
 }
 
 func (v SequenceValue) String() string {
@@ -413,13 +426,18 @@ func (p *parser) InjectItem(typ ItemType) {
 	p.injecting = true
 }
 
-func (*parser) newBinaryExpression(lhs Node, op Item, modifiers, rhs Node) *BinaryExpr {
+func (p *parser) newBinaryExpression(lhs Node, op Item, modifiers, rhs Node) *BinaryExpr {
 	ret := modifiers.(*BinaryExpr)
 
 	ret.LHS = lhs.(Expr)
 	ret.RHS = rhs.(Expr)
 	ret.Op = op.Typ
 
+	if !EnableBinopFillModifiers && (ret.VectorMatching.FillValues.LHS != nil || ret.VectorMatching.FillValues.RHS != nil) {
+		p.addParseErrf(ret.PositionRange(), "binop fill modifiers are experimental and not enabled")
+		return ret
+	}
+
 	return ret
 }
 
@@ -496,25 +514,30 @@ func (p *parser) mergeMaps(left, right *map[string]any) (ret *map[string]any) {
 }
 
 func (p *parser) histogramsIncreaseSeries(base, inc *histogram.FloatHistogram, times uint64) ([]SequenceValue, error) {
-	return p.histogramsSeries(base, inc, times, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
+	// Capture the hint set flag immediately after inc histogram is built.
+	// The base histogram's hint set flag was already captured.
+	hintSet := p.lastHistogramCounterResetHintSet
+	return p.histogramsSeries(base, inc, times, hintSet, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
 		res, _, _, err := a.Add(b)
 		return res, err
 	})
 }
 
 func (p *parser) histogramsDecreaseSeries(base, inc *histogram.FloatHistogram, times uint64) ([]SequenceValue, error) {
-	return p.histogramsSeries(base, inc, times, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
+	// Capture the hint set flag immediately after inc histogram is built.
+	hintSet := p.lastHistogramCounterResetHintSet
+	return p.histogramsSeries(base, inc, times, hintSet, func(a, b *histogram.FloatHistogram) (*histogram.FloatHistogram, error) {
 		res, _, _, err := a.Sub(b)
 		return res, err
 	})
 }
 
-func (*parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uint64,
+func (*parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uint64, counterResetHintSet bool,
 	combine func(*histogram.FloatHistogram, *histogram.FloatHistogram) (*histogram.FloatHistogram, error),
 ) ([]SequenceValue, error) {
 	ret := make([]SequenceValue, times+1)
 	// Add an additional value (the base) for time 0, which we ignore in tests.
-	ret[0] = SequenceValue{Histogram: base}
+	ret[0] = SequenceValue{Histogram: base, CounterResetHintSet: counterResetHintSet}
 	cur := base
 	for i := uint64(1); i <= times; i++ {
 		if cur.Schema > inc.Schema {
@@ -526,7 +549,7 @@ func (*parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uint6
 		if err != nil {
 			return ret, err
 		}
-		ret[i] = SequenceValue{Histogram: cur}
+		ret[i] = SequenceValue{Histogram: cur, CounterResetHintSet: counterResetHintSet}
 	}
 
 	return ret, nil
@@ -535,6 +558,8 @@ func (*parser) histogramsSeries(base, inc *histogram.FloatHistogram, times uint6
 // buildHistogramFromMap is used in the grammar to take then individual parts of the histogram and complete it.
 func (p *parser) buildHistogramFromMap(desc *map[string]any) *histogram.FloatHistogram {
 	output := &histogram.FloatHistogram{}
+	// Reset the flag for each new histogram being built.
+	p.lastHistogramCounterResetHintSet = false
 
 	val, ok := (*desc)["schema"]
 	if ok {
@@ -595,6 +620,8 @@ func (p *parser) buildHistogramFromMap(desc *map[string]any) *histogram.FloatHis
 
 	val, ok = (*desc)["counter_reset_hint"]
 	if ok {
+		// Mark that the counter reset hint was explicitly specified.
+		p.lastHistogramCounterResetHintSet = true
 		resetHint, ok := val.(Item)
 
 		if ok {
@@ -626,6 +653,16 @@ func (p *parser) buildHistogramFromMap(desc *map[string]any) *histogram.FloatHis
 	return output
 }
 
+// newHistogramSequenceValue creates a SequenceValue for a histogram,
+// setting CounterResetHintSet based on whether counter_reset_hint was
+// explicitly specified in the histogram description.
+func (p *parser) newHistogramSequenceValue(h *histogram.FloatHistogram) SequenceValue {
+	return SequenceValue{
+		Histogram:           h,
+		CounterResetHintSet: p.lastHistogramCounterResetHintSet,
+	}
+}
+
 func (p *parser) buildHistogramBucketsAndSpans(desc *map[string]any, bucketsKey, offsetKey string,
 ) (buckets []float64, spans []histogram.Span) {
 	bucketCount := 0
@@ -768,6 +805,9 @@ func (p *parser) checkAST(node Node) (typ ValueType) {
 			if len(n.VectorMatching.MatchingLabels) > 0 {
 				p.addParseErrf(n.PositionRange(), "vector matching only allowed between instant vectors")
 			}
+			if n.VectorMatching.FillValues.LHS != nil || n.VectorMatching.FillValues.RHS != nil {
+				p.addParseErrf(n.PositionRange(), "filling in missing series only allowed between instant vectors")
+			}
 			n.VectorMatching = nil
 		case n.Op.IsSetOperator(): // Both operands are Vectors.
 			if n.VectorMatching.Card == CardOneToMany || n.VectorMatching.Card == CardManyToOne {
@@ -776,6 +816,9 @@ func (p *parser) checkAST(node Node) (typ ValueType) {
 			if n.VectorMatching.Card != CardManyToMany {
 				p.addParseErrf(n.PositionRange(), "set operations must always be many-to-many")
 			}
+			if n.VectorMatching.FillValues.LHS != nil || n.VectorMatching.FillValues.RHS != nil {
+				p.addParseErrf(n.PositionRange(), "filling in missing series not allowed for set operators")
+			}
 		}
 
 		if (lt == ValueTypeScalar || rt == ValueTypeScalar) && n.Op.IsSetOperator() {

promql/parser/printer.go

@@ -172,6 +172,19 @@ func (node *BinaryExpr) getMatchingStr() string {
 			b.WriteString(")")
 			matching += b.String()
 		}
+
+		if vm.FillValues.LHS != nil || vm.FillValues.RHS != nil {
+			if vm.FillValues.LHS == vm.FillValues.RHS {
+				matching += fmt.Sprintf(" fill (%v)", *vm.FillValues.LHS)
+			} else {
+				if vm.FillValues.LHS != nil {
+					matching += fmt.Sprintf(" fill_left (%v)", *vm.FillValues.LHS)
+				}
+				if vm.FillValues.RHS != nil {
+					matching += fmt.Sprintf(" fill_right (%v)", *vm.FillValues.RHS)
+				}
+			}
+		}
 	}
 	return matching
 }

promql/parser/printer_test.go

@@ -23,8 +23,10 @@ import (
 
 func TestExprString(t *testing.T) {
 	ExperimentalDurationExpr = true
+	EnableBinopFillModifiers = true
 	t.Cleanup(func() {
 		ExperimentalDurationExpr = false
+		EnableBinopFillModifiers = false
 	})
 	// A list of valid expressions that are expected to be
 	// returned as out when calling String(). If out is empty the output
@@ -113,6 +115,26 @@ func TestExprString(t *testing.T) {
 			in:  `a - ignoring() group_left c`,
 			out: `a - ignoring () group_left () c`,
 		},
+		{
+			in:  `a + fill(-23) b`,
+			out: `a + fill (-23) b`,
+		},
+		{
+			in:  `a + fill_left(-23) b`,
+			out: `a + fill_left (-23) b`,
+		},
+		{
+			in:  `a + fill_right(42) b`,
+			out: `a + fill_right (42) b`,
+		},
+		{
+			in:  `a + fill_left(-23) fill_right(42) b`,
+			out: `a + fill_left (-23) fill_right (42) b`,
+		},
+		{
+			in:  `a + on(b) group_left fill(-23) c`,
+			out: `a + on (b) group_left () fill (-23) c`,
+		},
 		{
 			in: `up > bool 0`,
 		},

promql/promqltest/test.go

@@ -43,7 +43,6 @@ import (
 	"github.com/prometheus/prometheus/util/annotations"
 	"github.com/prometheus/prometheus/util/convertnhcb"
 	"github.com/prometheus/prometheus/util/teststorage"
-	"github.com/prometheus/prometheus/util/testutil"
 )
 
 var (
@@ -72,7 +71,7 @@ var testStartTime = time.Unix(0, 0).UTC()
 
 // LoadedStorage returns storage with generated data using the provided load statements.
 // Non-load statements will cause test errors.
-func LoadedStorage(t testutil.T, input string) *teststorage.TestStorage {
+func LoadedStorage(t testing.TB, input string) *teststorage.TestStorage {
 	test, err := newTest(t, input, false, newTestStorage)
 	require.NoError(t, err)
 
@@ -113,21 +112,56 @@ func NewTestEngineWithOpts(tb testing.TB, opts promql.EngineOpts) *promql.Engine
 	return ng
 }
 
+// GetBuiltInExprs returns all the eval statement expressions from the built-in test files.
+func GetBuiltInExprs() ([]string, error) {
+	files, err := fs.Glob(testsFs, "*/*.test")
+	if err != nil {
+		return nil, err
+	}
+
+	var exprs []string
+	for _, fn := range files {
+		content, err := fs.ReadFile(testsFs, fn)
+		if err != nil {
+			return nil, err
+		}
+
+		// Create a minimal test struct just for parsing
+		testInstance := &test{
+			cmds: []testCommand{},
+		}
+		if err := testInstance.parse(string(content)); err != nil {
+			return nil, err
+		}
+
+		// Extract expressions from eval commands
+		for _, cmd := range testInstance.cmds {
+			if evalCmd, ok := cmd.(*evalCmd); ok {
+				exprs = append(exprs, evalCmd.expr)
+			}
+		}
+	}
+
+	return exprs, nil
+}
+
 // RunBuiltinTests runs an acceptance test suite against the provided engine.
 func RunBuiltinTests(t TBRun, engine promql.QueryEngine) {
 	RunBuiltinTestsWithStorage(t, engine, newTestStorage)
 }
 
 // RunBuiltinTestsWithStorage runs an acceptance test suite against the provided engine and storage.
-func RunBuiltinTestsWithStorage(t TBRun, engine promql.QueryEngine, newStorage func(testutil.T) storage.Storage) {
+func RunBuiltinTestsWithStorage(t TBRun, engine promql.QueryEngine, newStorage func(testing.TB) storage.Storage) {
 	t.Cleanup(func() {
 		parser.EnableExperimentalFunctions = false
 		parser.ExperimentalDurationExpr = false
 		parser.EnableExtendedRangeSelectors = false
+		parser.EnableBinopFillModifiers = false
 	})
 	parser.EnableExperimentalFunctions = true
 	parser.ExperimentalDurationExpr = true
 	parser.EnableExtendedRangeSelectors = true
+	parser.EnableBinopFillModifiers = true
 
 	files, err := fs.Glob(testsFs, "*/*.test")
 	require.NoError(t, err)
@@ -142,22 +176,22 @@ func RunBuiltinTestsWithStorage(t TBRun, engine promql.QueryEngine, newStorage f
 }
 
 // RunTest parses and runs the test against the provided engine.
-func RunTest(t testutil.T, input string, engine promql.QueryEngine) {
+func RunTest(t testing.TB, input string, engine promql.QueryEngine) {
 	RunTestWithStorage(t, input, engine, newTestStorage)
 }
 
 // RunTestWithStorage parses and runs the test against the provided engine and storage.
-func RunTestWithStorage(t testutil.T, input string, engine promql.QueryEngine, newStorage func(testutil.T) storage.Storage) {
+func RunTestWithStorage(t testing.TB, input string, engine promql.QueryEngine, newStorage func(testing.TB) storage.Storage) {
 	require.NoError(t, runTest(t, input, engine, newStorage, false))
 }
 
 // testTest allows tests to be run in "test-the-test" mode (true for
 // testingMode). This is a special mode for testing test code execution itself.
-func testTest(t testutil.T, input string, engine promql.QueryEngine) error {
+func testTest(t testing.TB, input string, engine promql.QueryEngine) error {
 	return runTest(t, input, engine, newTestStorage, true)
 }
 
-func runTest(t testutil.T, input string, engine promql.QueryEngine, newStorage func(testutil.T) storage.Storage, testingMode bool) error {
+func runTest(t testing.TB, input string, engine promql.QueryEngine, newStorage func(testing.TB) storage.Storage, testingMode bool) error {
 	test, err := newTest(t, input, testingMode, newStorage)
 
 	// Why do this before checking err? newTest() can create the test storage and then return an error,
@@ -192,13 +226,14 @@ func runTest(t testutil.T, input string, engine promql.QueryEngine, newStorage f
 // test is a sequence of read and write commands that are run
 // against a test storage.
 type test struct {
-	testutil.T
+	testing.TB
+
 	// testingMode distinguishes between normal execution and test-execution mode.
 	testingMode bool
 
 	cmds []testCommand
 
-	open    func(testutil.T) storage.Storage
+	open    func(testing.TB) storage.Storage
 	storage storage.Storage
 
 	context   context.Context
@@ -206,9 +241,9 @@ type test struct {
 }
 
 // newTest returns an initialized empty Test.
-func newTest(t testutil.T, input string, testingMode bool, newStorage func(testutil.T) storage.Storage) (*test, error) {
+func newTest(t testing.TB, input string, testingMode bool, newStorage func(testing.TB) storage.Storage) (*test, error) {
 	test := &test{
-		T:           t,
+		TB:          t,
 		cmds:        []testCommand{},
 		testingMode: testingMode,
 		open:        newStorage,
@@ -219,7 +254,7 @@ func newTest(t testutil.T, input string, testingMode bool, newStorage func(testu
 	return test, err
 }
 
-func newTestStorage(t testutil.T) storage.Storage { return teststorage.New(t) }
+func newTestStorage(t testing.TB) storage.Storage { return teststorage.New(t) }
 
 //go:embed testdata
 var testsFs embed.FS
@@ -1012,7 +1047,12 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 			exp := ev.expected[hash]
 
 			var expectedFloats []promql.FPoint
-			var expectedHistograms []promql.HPoint
+			// expectedHPoint wraps HPoint with CounterResetHintSet flag from SequenceValue.
+			type expectedHPoint struct {
+				promql.HPoint
+				CounterResetHintSet bool
+			}
+			var expectedHistograms []expectedHPoint
 
 			for i, e := range exp.vals {
 				ts := ev.start.Add(time.Duration(i) * ev.step)
@@ -1024,7 +1064,10 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 				t := ts.UnixNano() / int64(time.Millisecond/time.Nanosecond)
 
 				if e.Histogram != nil {
-					expectedHistograms = append(expectedHistograms, promql.HPoint{T: t, H: e.Histogram})
+					expectedHistograms = append(expectedHistograms, expectedHPoint{
+						HPoint:              promql.HPoint{T: t, H: e.Histogram},
+						CounterResetHintSet: e.CounterResetHintSet,
+					})
 				} else if !e.Omitted {
 					expectedFloats = append(expectedFloats, promql.FPoint{T: t, F: e.Value})
 				}
@@ -1053,7 +1096,7 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 					return fmt.Errorf("expected histogram value at index %v for %s to have timestamp %v, but it had timestamp %v (result has %s)", i, ev.metrics[hash], expected.T, actual.T, formatSeriesResult(s))
 				}
 
-				if !compareNativeHistogram(expected.H.Compact(0), actual.H.Compact(0)) {
+				if !compareNativeHistogram(expected.H.Compact(0), actual.H.Compact(0), expected.CounterResetHintSet) {
 					return fmt.Errorf("expected histogram value at index %v (t=%v) for %s to be %v, but got %v (result has %s)", i, actual.T, ev.metrics[hash], expected.H.TestExpression(), actual.H.TestExpression(), formatSeriesResult(s))
 				}
 			}
@@ -1092,7 +1135,7 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 			if expH != nil && v.H == nil {
 				return fmt.Errorf("expected histogram %s for %s but got float value %v", HistogramTestExpression(expH), v.Metric, v.F)
 			}
-			if expH != nil && !compareNativeHistogram(expH.Compact(0), v.H.Compact(0)) {
+			if expH != nil && !compareNativeHistogram(expH.Compact(0), v.H.Compact(0), exp0.CounterResetHintSet) {
 				return fmt.Errorf("expected %v for %s but got %s", HistogramTestExpression(expH), v.Metric, HistogramTestExpression(v.H))
 			}
 			if !almost.Equal(exp0.Value, v.F, defaultEpsilon) {
@@ -1130,7 +1173,9 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 
 // compareNativeHistogram is helper function to compare two native histograms
 // which can tolerate some differ in the field of float type, such as Count, Sum.
-func compareNativeHistogram(exp, cur *histogram.FloatHistogram) bool {
+// The counterResetHintSet parameter indicates whether the counter reset hint was
+// explicitly specified in the expected histogram (from the test file).
+func compareNativeHistogram(exp, cur *histogram.FloatHistogram, counterResetHintSet bool) bool {
 	if exp == nil || cur == nil {
 		return false
 	}
@@ -1166,6 +1211,15 @@ func compareNativeHistogram(exp, cur *histogram.FloatHistogram) bool {
 		return false
 	}
 
+	// Compare CounterResetHint only if explicitly specified in expected histogram.
+	// When counterResetHintSet is false, no hint was specified, meaning "don't care".
+	// When counterResetHintSet is true, the hint was explicitly specified and must match.
+	if counterResetHintSet {
+		if exp.CounterResetHint != cur.CounterResetHint {
+			return false
+		}
+	}
+
 	return true
 }
 
@@ -1421,7 +1475,7 @@ func (t *test) execEval(cmd *evalCmd, engine promql.QueryEngine) error {
 		return do()
 	}
 
-	if tt, ok := t.T.(*testing.T); ok {
+	if tt, ok := t.TB.(*testing.T); ok {
 		tt.Run(fmt.Sprintf("line %d/%s", cmd.line, cmd.expr), func(t *testing.T) {
 			require.NoError(t, do())
 		})
@@ -1589,12 +1643,12 @@ func assertMatrixSorted(m promql.Matrix) error {
 func (t *test) clear() {
 	if t.storage != nil {
 		err := t.storage.Close()
-		require.NoError(t.T, err, "Unexpected error while closing test storage.")
+		require.NoError(t.TB, err, "Unexpected error while closing test storage.")
 	}
 	if t.cancelCtx != nil {
 		t.cancelCtx()
 	}
-	t.storage = t.open(t.T)
+	t.storage = t.open(t.TB)
 	t.context, t.cancelCtx = context.WithCancel(context.Background())
 }
 

promql/promqltest/testdata/fill-modifier.test

@@ -0,0 +1,383 @@
+# ==================== fill / fill_left / fill_right modifier tests ====================
+
+# Test data for fill modifier tests: vectors with partial overlap.
+load 5m
+  left_vector{label="a"} 10
+  left_vector{label="b"} 20
+  left_vector{label="c"} 30
+  right_vector{label="a"} 100
+  right_vector{label="b"} 200
+  right_vector{label="d"} 400
+
+# ---------- Arithmetic operators with fill modifiers ----------
+
+# fill(0): Fill both sides with 0 for addition.
+eval instant at 0m left_vector + fill(0) right_vector
+  {label="a"} 110
+  {label="b"} 220
+  {label="c"} 30
+  {label="d"} 400
+
+# fill_left(0): Only fill left side with 0.
+eval instant at 0m left_vector + fill_left(0) right_vector
+  {label="a"} 110
+  {label="b"} 220
+  {label="d"} 400
+
+# fill_right(0): Only fill right side with 0.
+eval instant at 0m left_vector + fill_right(0) right_vector
+  {label="a"} 110
+  {label="b"} 220
+  {label="c"} 30
+
+# fill_left and fill_right with different values.
+eval instant at 0m left_vector + fill_left(5) fill_right(7) right_vector
+  {label="a"} 110
+  {label="b"} 220
+  {label="c"} 37
+  {label="d"} 405
+
+# fill with NaN.
+eval instant at 0m left_vector + fill(NaN) right_vector
+  {label="a"} 110
+  {label="b"} 220
+  {label="c"} NaN
+  {label="d"} NaN
+
+# fill with Inf.
+eval instant at 0m left_vector + fill(Inf) right_vector
+  {label="a"} 110
+  {label="b"} 220
+  {label="c"} +Inf
+  {label="d"} +Inf
+
+# fill with -Inf.
+eval instant at 0m left_vector + fill(-Inf) right_vector
+  {label="a"} 110
+  {label="b"} 220
+  {label="c"} -Inf
+  {label="d"} -Inf
+
+# ---------- Comparison operators with fill modifiers ----------
+
+# fill with equality comparison.
+eval instant at 0m left_vector == fill(30) right_vector
+  left_vector{label="c"} 30
+
+# fill with inequality comparison.
+eval instant at 0m left_vector != fill(30) right_vector
+  left_vector{label="a"} 10
+  left_vector{label="b"} 20
+  {label="d"} 30
+
+# fill with greater than.
+eval instant at 0m left_vector > fill(25) right_vector
+  left_vector{label="c"} 30
+
+# ---------- Comparison operators with bool modifier and fill ----------
+
+# fill with equality comparison and bool.
+eval instant at 0m left_vector == bool fill(30) right_vector
+  {label="a"} 0
+  {label="b"} 0
+  {label="c"} 1
+  {label="d"} 0
+
+# fill with inequality comparison and bool.
+eval instant at 0m left_vector != bool fill(30) right_vector
+  {label="a"} 1
+  {label="b"} 1
+  {label="c"} 0
+  {label="d"} 1
+
+# fill with greater than and bool.
+eval instant at 0m left_vector > bool fill(25) right_vector
+  {label="a"} 0
+  {label="b"} 0
+  {label="c"} 1
+  {label="d"} 0
+
+# ---------- fill with on() and ignoring() modifiers ----------
+
+clear
+
+load 5m
+  left_vector{job="foo", instance="a"} 10
+  left_vector{job="foo", instance="b"} 20
+  left_vector{job="bar", instance="a"} 30
+  right_vector{job="foo", instance="a"} 100
+  right_vector{job="foo", instance="c"} 300
+
+# fill with on().
+eval instant at 0m left_vector + on(job, instance) fill(0) right_vector
+  {job="foo", instance="a"} 110
+  {job="foo", instance="b"} 20
+  {job="bar", instance="a"} 30
+  {job="foo", instance="c"} 300
+
+# fill_right with on().
+eval instant at 0m left_vector + on(job, instance) fill_right(0) right_vector
+  {job="foo", instance="a"} 110
+  {job="foo", instance="b"} 20
+  {job="bar", instance="a"} 30
+
+# fill_left with on().
+eval instant at 0m left_vector + on(job, instance) fill_left(0) right_vector
+  {job="foo", instance="a"} 110
+  {job="foo", instance="c"} 300
+
+# fill with ignoring() - requires group_left since ignoring(job) creates many-to-one matching
+# when two left_vector series have same instance but different jobs.
+eval instant at 0m left_vector + ignoring(job) group_left fill(0) right_vector
+  {instance="a", job="foo"} 110
+  {instance="a", job="bar"} 130
+  {instance="b", job="foo"} 20
+  {instance="c"} 300
+
+# ---------- fill with group_left / group_right (many-to-one / one-to-many) ----------
+
+clear
+
+load 5m
+  requests{method="GET", status="200"} 100
+  requests{method="POST", status="200"} 200
+  requests{method="GET", status="500"} 10
+  requests{method="POST", status="500"} 20
+  limits{status="200"} 1000
+  limits{status="404"} 500
+  limits{status="500"} 50
+
+# group_left with fill_right: fill missing "one" side series.
+eval instant at 0m requests / on(status) group_left fill_right(1) limits
+  {method="GET", status="200"} 0.1
+  {method="POST", status="200"} 0.2
+  {method="GET", status="500"} 0.2
+  {method="POST", status="500"} 0.4
+
+# group_left with fill_left: fill missing "many" side series.
+# For status="404", there's no matching requests, so a single series with the match group's labels is filled
+eval instant at 0m requests + on(status) group_left fill_left(0) limits
+  {method="GET", status="200"} 1100
+  {method="POST", status="200"} 1200
+  {method="GET", status="500"} 60
+  {method="POST", status="500"} 70
+  {status="404"} 500
+
+# group_left with fill on both sides.
+eval instant at 0m requests + on(status) group_left fill(0) limits
+  {method="GET", status="200"} 1100
+  {method="POST", status="200"} 1200
+  {method="GET", status="500"} 60
+  {method="POST", status="500"} 70
+  {status="404"} 500
+
+# group_right with fill_left: fill missing "one" side series.
+clear
+
+load 5m
+  cpu_info{instance="a", cpu="0"} 1
+  cpu_info{instance="a", cpu="1"} 1
+  cpu_info{instance="b", cpu="0"} 1
+  node_meta{instance="a"} 100
+  node_meta{instance="c"} 300
+
+# fill_left fills the "one" side (node_meta) when missing for a "many" side series.
+eval instant at 0m node_meta * on(instance) group_right fill_left(1) cpu_info
+  {instance="a", cpu="0"} 100
+  {instance="a", cpu="1"} 100
+  {instance="c"} 300
+
+# group_right with fill_right: fill missing "many" side series.
+eval instant at 0m node_meta * on(instance) group_right fill_right(0) cpu_info
+  {instance="a", cpu="0"} 100
+  {instance="a", cpu="1"} 100
+  {instance="b", cpu="0"} 0
+
+# group_right with fill on both sides.
+eval instant at 0m node_meta * on(instance) group_right fill(1) cpu_info
+  {instance="a", cpu="0"} 100
+  {instance="a", cpu="1"} 100
+  {instance="b", cpu="0"} 1
+  {instance="c"} 300
+
+# ---------- fill with group_left/group_right and extra labels ----------
+
+clear
+
+load 5m
+  requests{method="GET", status="200"} 100
+  requests{method="POST", status="200"} 200
+  limits{status="200", owner="team-a"} 1000
+  limits{status="500", owner="team-b"} 50
+
+# group_left with extra label and fill_right.
+# Note: when filling the "one" side, the joined label cannot be filled.
+eval instant at 0m requests + on(status) group_left(owner) fill_right(0) limits
+  {method="GET", status="200", owner="team-a"} 1100
+  {method="POST", status="200", owner="team-a"} 1200
+
+# ---------- Edge cases ----------
+
+clear
+
+load 5m
+  only_left{label="a"} 10
+  only_left{label="b"} 20
+  only_right{label="c"} 30
+  only_right{label="d"} 40
+
+# No overlap at all - fill creates all results.
+eval instant at 0m only_left + fill(0) only_right
+  {label="a"} 10
+  {label="b"} 20
+  {label="c"} 30
+  {label="d"} 40
+
+# No overlap - fill_left only creates right side results.
+eval instant at 0m only_left + fill_left(0) only_right
+  {label="c"} 30
+  {label="d"} 40
+
+# No overlap - fill_right only creates left side results.
+eval instant at 0m only_left + fill_right(0) only_right
+  {label="a"} 10
+  {label="b"} 20
+
+# Complete overlap - fill has no effect.
+clear
+
+load 5m
+  complete_left{label="a"} 10
+  complete_left{label="b"} 20
+  complete_right{label="a"} 100
+  complete_right{label="b"} 200
+
+eval instant at 0m complete_left + fill(99) complete_right
+  {label="a"} 110
+  {label="b"} 220
+
+# ---------- fill with range queries ----------
+
+clear
+
+load 5m
+  range_left{label="a"} 1 2 3 4 5
+  range_left{label="b"} 10 20 30 40 50
+  range_right{label="a"} 100 200 300 400 500
+  range_right{label="c"} 1000 2000 3000 4000 5000
+
+eval range from 0 to 20m step 5m range_left + fill(0) range_right
+  {label="a"} 101 202 303 404 505
+  {label="b"} 10 20 30 40 50
+  {label="c"} 1000 2000 3000 4000 5000
+
+eval range from 0 to 20m step 5m range_left + fill_right(0) range_right
+  {label="a"} 101 202 303 404 505
+  {label="b"} 10 20 30 40 50
+
+eval range from 0 to 20m step 5m range_left + fill_left(0) range_right
+  {label="a"} 101 202 303 404 505
+  {label="c"} 1000 2000 3000 4000 5000
+
+# Range queries with intermittently present series.
+clear
+
+load 5m
+  intermittent_left{label="a"}  1  _  3  _  5
+  intermittent_left{label="b"}  _  20 _  40 _
+  intermittent_right{label="a"} _  200 _ 400 _
+  intermittent_right{label="b"} 100 _ 300 _ 500
+  intermittent_right{label="c"} 1000 _ _ 4000 5000
+
+# When both sides have the same label but are present at different times,
+# fill creates results at all timestamps where at least one side is present.
+eval range from 0 to 20m step 5m intermittent_left + fill(0) intermittent_right
+  {label="a"} 1 200 3 400 5
+  {label="b"} 100 20 300 40 500
+  {label="c"} 1000 _ _ 4000 5000
+
+# fill_right only fills the right side when it's missing.
+# Output only exists when left side is present (right side filled with 0 if missing).
+eval range from 0 to 20m step 5m intermittent_left + fill_right(0) intermittent_right
+  {label="a"} 1 _ 3 _ 5
+  {label="b"} _ 20 _ 40 _
+
+# fill_left only fills the left side when it's missing.
+# Output only exists when right side is present (left side filled with 0 if missing).
+eval range from 0 to 20m step 5m intermittent_left + fill_left(0) intermittent_right
+  {label="a"} _ 200 _ 400 _
+  {label="b"} 100 _ 300 _ 500
+  {label="c"} 1000 _ _ 4000 5000
+
+# ---------- fill with vectors where one side is empty ----------
+
+clear
+
+load 5m
+  non_empty{label="a"} 10
+  non_empty{label="b"} 20
+
+# Empty right side - fill_right has no effect (nothing to add).
+eval instant at 0m non_empty + fill_right(0) nonexistent
+  {label="a"} 10
+  {label="b"} 20
+
+# Empty right side - fill_left creates nothing (no right side labels to use).
+eval instant at 0m non_empty + fill_left(0) nonexistent
+
+# Empty left side - fill_left has no effect.
+eval instant at 0m nonexistent + fill_left(0) non_empty
+  {label="a"} 10
+  {label="b"} 20
+
+# Empty left side - fill_right creates nothing.
+eval instant at 0m nonexistent + fill_right(0) non_empty
+
+# fill both sides with one side empty.
+eval instant at 0m non_empty + fill(0) nonexistent
+  {label="a"} 10
+  {label="b"} 20
+
+eval instant at 0m nonexistent + fill(0) non_empty
+  {label="a"} 10
+  {label="b"} 20
+
+# ---------- Metric names that match fill modifier keywords ----------
+
+clear
+
+load 5m
+  fill{label="a"} 1
+  fill{label="b"} 2
+  fill_left{label="a"} 10
+  fill_left{label="c"} 30
+  fill_right{label="b"} 200
+  fill_right{label="d"} 400
+  other{label="a"} 1000
+  other{label="e"} 5000
+
+# Metric named "fill" on the left side.
+eval instant at 0m fill + fill(0) other
+  {label="a"} 1001
+  {label="b"} 2
+  {label="e"} 5000
+
+# Metric named "fill" on the right side without modifier.
+eval instant at 0m other + fill
+  {label="a"} 1001
+
+# Metric named "fill" on the right side with fill() modifier.
+eval instant at 0m other + fill(0) fill
+  {label="a"} 1001
+  {label="b"} 2
+  {label="e"} 5000
+
+# Metric named "fill_left" on the right side with fill_left() modifier.
+eval instant at 0m other + fill_left(0) fill_left
+  {label="a"} 1010
+  {label="c"} 30
+
+# Metric named "fill_right" on the right side with fill_right() modifier.
+eval instant at 0m other + fill_right(0) fill_right
+  {label="a"} 1000
+  {label="e"} 5000

promql/promqltest/testdata/info.test

@@ -34,6 +34,22 @@ eval range from 0m to 10m step 5m info(metric, {data=~".+", non_existent=~".*"})
 eval range from 0m to 10m step 5m info(metric_with_overlapping_label)
     metric_with_overlapping_label{data="base", instance="a", job="1", label="value", another_data="another info"} 0 1 2
 
+# Filtering by a label that exists on both base metric and target_info should work.
+# This is a regression test for https://github.com/prometheus/prometheus/issues/17813.
+# Note: data="base" on base metric, data="info" on target_info - the filter matches target_info.
+eval range from 0m to 10m step 5m info(metric_with_overlapping_label, {data="info"})
+    metric_with_overlapping_label{data="base", instance="a", job="1", label="value"} 0 1 2
+
+# Filtering by a label that exists on both base metric and target_info with regex should work.
+eval range from 0m to 10m step 5m info(metric_with_overlapping_label, {data=~".+"})
+    metric_with_overlapping_label{data="base", instance="a", job="1", label="value"} 0 1 2
+
+# Filtering by a label that exists on both base metric and target_info with same value.
+# The selector matches the target_info, and the join succeeds via identifying labels.
+# Note: Only the instance label is considered for inclusion, but it already exists on base.
+eval range from 0m to 10m step 5m info(metric_with_overlapping_label, {instance="a"})
+    metric_with_overlapping_label{data="base", instance="a", job="1", label="value"} 0 1 2
+
 # Include data labels from target_info specifically.
 eval range from 0m to 10m step 5m info(metric, {__name__="target_info"})
     metric{data="info", instance="a", job="1", label="value", another_data="another info"} 0 1 2
@@ -54,7 +70,11 @@ eval range from 0m to 10m step 5m info(metric, {__name__=~".+_info"})
     metric{instance="a", job="1", label="value", build_data="build", data="info", another_data="another info"} 0 1 2
 
 # Info metrics themselves are ignored when it comes to enriching with info metric data labels.
-eval range from 0m to 10m step 5m info(build_info, {__name__=~".+_info", build_data=~".+"})
+eval range from 0m to 10m step 5m info(build_info, {__name__=~".+_info", another_data=~".+"})
+    build_info{instance="a", job="1", build_data="build"} 1 1 1
+
+# Info metrics themselves are ignored when it comes to enriching with info metric data labels.
+eval range from 0m to 10m step 5m info(build_info, {__name__=~".+_info"})
     build_info{instance="a", job="1", build_data="build"} 1 1 1
 
 clear
@@ -150,3 +170,35 @@ eval range from 0 to 2m step 1m info({job="work"}, {__name__="info_metric"})
 	data_metric{instance="a", job="work", state="running", label="new"} _ _ 30
 	info_metric{instance="b", job="work", state="stopped"} 1 1 1
 	info_metric{instance="a", job="work", state="running"} 1 1 1
+
+clear
+
+load 1m
+	data_metric{} 1 2 3
+
+eval range from 0 to 2m step 1m info(data_metric, {__name__="info_metric"})
+	data_metric{} 1 2 3
+
+clear
+
+load 1m
+	data_metric{} 1 2 3
+	data_metric{instance="a"} 4 5 6
+
+eval range from 0 to 2m step 1m info(data_metric, {__name__="info_metric"})
+	data_metric{} 1 2 3
+	data_metric{instance="a"} 4 5 6
+
+clear
+
+load 1m
+	data_metric{} 1 2 3
+	data_metric{instance="a"} 4 5 6
+	data_metric{job="1"} 7 8 9
+	data_metric{instance="a", job="1"} 10 20 30
+
+eval range from 0 to 2m step 1m info(data_metric, {__name__="info_metric"})
+	data_metric{} 1 2 3
+	data_metric{instance="a"} 4 5 6
+	data_metric{job="1"} 7 8 9
+	data_metric{instance="a", job="1"} 10 20 30

promql/promqltest/testdata/native_histograms.test

@@ -1283,7 +1283,7 @@ eval instant at 12m sum_over_time(nhcb_metric[13m])
 eval instant at 12m avg_over_time(nhcb_metric[13m])
   expect no_warn
   expect info msg: PromQL info: mismatched custom buckets were reconciled during aggregation
-  {} {{schema:-53 count:1 sum:1 custom_values:[5] counter_reset_hint:gauge buckets:[1]}}
+  {} {{schema:-53 count:1 sum:1 custom_values:[5] buckets:[1]}}
 
 eval instant at 12m last_over_time(nhcb_metric[13m])
   expect no_warn

promql/value.go

@@ -487,6 +487,11 @@ func (ssi *storageSeriesIterator) AtT() int64 {
 	return ssi.currT
 }
 
+// TODO(krajorama): implement AtST.
+func (*storageSeriesIterator) AtST() int64 {
+	return 0
+}
+
 func (ssi *storageSeriesIterator) Next() chunkenc.ValueType {
 	if ssi.currH != nil {
 		ssi.iHistograms++

rules/alerting_test.go

@@ -697,12 +697,14 @@ func TestQueryForStateSeries(t *testing.T) {
 		{
 			selectMockFunction: func(bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet {
 				return storage.TestSeriesSet(storage.MockSeries(
+					nil,
 					[]int64{1, 2, 3},
 					[]float64{1, 2, 3},
 					[]string{"__name__", "ALERTS_FOR_STATE", "alertname", "TestRule", "severity", "critical"},
 				))
 			},
 			expectedSeries: storage.MockSeries(
+				nil,
 				[]int64{1, 2, 3},
 				[]float64{1, 2, 3},
 				[]string{"__name__", "ALERTS_FOR_STATE", "alertname", "TestRule", "severity", "critical"},

rules/manager_test.go

@@ -45,6 +45,7 @@ import (
 	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/prometheus/prometheus/promql/promqltest"
 	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/tsdbutil"
 	"github.com/prometheus/prometheus/util/teststorage"
@@ -1201,7 +1202,9 @@ func TestRuleMovedBetweenGroups(t *testing.T) {
 		t.Skip("skipping test in short mode.")
 	}
 
-	storage := teststorage.New(t, 600000)
+	storage := teststorage.New(t, func(opt *tsdb.Options) {
+		opt.OutOfOrderTimeWindow = 600000
+	})
 	defer storage.Close()
 	opts := promql.EngineOpts{
 		Logger:     nil,

rules/recording_test.go

@@ -111,7 +111,7 @@ var ruleEvalTestScenarios = []struct {
 	},
 }
 
-func setUpRuleEvalTest(t require.TestingT) *teststorage.TestStorage {
+func setUpRuleEvalTest(t testing.TB) *teststorage.TestStorage {
 	return promqltest.LoadedStorage(t, `
 		load 1m
 			metric{label_a="1",label_b="3"} 1

scrape/helpers_test.go

@@ -17,6 +17,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"fmt"
 	"net/http"
 	"testing"
 	"time"
@@ -38,15 +39,22 @@ import (
 // For readability.
 type sample = teststorage.Sample
 
+type compatAppendable interface {
+	storage.Appendable
+	storage.AppendableV2
+}
+
 func withCtx(ctx context.Context) func(sl *scrapeLoop) {
 	return func(sl *scrapeLoop) {
 		sl.ctx = ctx
 	}
 }
 
-func withAppendable(appendable storage.Appendable) func(sl *scrapeLoop) {
+func withAppendable(app compatAppendable, appV2 bool) func(sl *scrapeLoop) {
 	return func(sl *scrapeLoop) {
-		sl.appendable = appendable
+		sa := selectAppendable(app, appV2)
+		sl.appendable = sa.V1()
+		sl.appendableV2 = sa.V2()
 	}
 }
 
@@ -55,8 +63,7 @@ func withAppendable(appendable storage.Appendable) func(sl *scrapeLoop) {
 //
 // It's recommended to use withXYZ functions for simple option customizations, e.g:
 //
-//	appTest := teststorage.NewAppendable()
-//	sl, _ := newTestScrapeLoop(t, withAppendable(appTest))
+//	sl, _ := newTestScrapeLoop(t, withCtx(customCtx))
 //
 // However, when changing more than one scrapeLoop options it's more readable to have one explicit opt function:
 //
@@ -64,7 +71,7 @@ func withAppendable(appendable storage.Appendable) func(sl *scrapeLoop) {
 //	appTest := teststorage.NewAppendable()
 //	sl, scraper := newTestScrapeLoop(t, func(sl *scrapeLoop) {
 //		sl.ctx = ctx
-//		sl.appendable = appTest
+//		sl.appendableV2 = appTest
 //		// Since we're writing samples directly below we need to provide a protocol fallback.
 //		sl.fallbackScrapeProtocol = "text/plain"
 //	})
@@ -84,8 +91,6 @@ func newTestScrapeLoop(t testing.TB, opts ...func(sl *scrapeLoop)) (_ *scrapeLoo
 		timeout:             1 * time.Hour,
 		sampleMutator:       nopMutator,
 		reportSampleMutator: nopMutator,
-
-		appendable:          teststorage.NewAppendable(),
 		buffers:             pool.New(1e3, 1e6, 3, func(sz int) any { return make([]byte, 0, sz) }),
 		metrics:             metrics,
 		maxSchema:           histogram.ExponentialSchemaMax,
@@ -98,6 +103,11 @@ func newTestScrapeLoop(t testing.TB, opts ...func(sl *scrapeLoop)) (_ *scrapeLoo
 	for _, o := range opts {
 		o(sl)
 	}
+
+	if sl.appendable != nil && sl.appendableV2 != nil {
+		t.Fatal("select the appendable to use, both were passed, likely a bug")
+	}
+
 	// Validate user opts for convenience.
 	require.Nil(t, sl.parentCtx, "newTestScrapeLoop does not support injecting non-nil parent context")
 	require.Nil(t, sl.appenderCtx, "newTestScrapeLoop does not support injecting non-nil appender context")
@@ -121,7 +131,8 @@ func newTestScrapeLoop(t testing.TB, opts ...func(sl *scrapeLoop)) (_ *scrapeLoo
 	return sl, scraper
 }
 
-func newTestScrapePool(t *testing.T, injectNewLoop func(options scrapeLoopOptions) loop) *scrapePool {
+func newTestScrapePool(t *testing.T, app compatAppendable, appV2 bool, injectNewLoop func(options scrapeLoopOptions) loop) *scrapePool {
+	sa := selectAppendable(app, appV2)
 	return &scrapePool{
 		ctx:     t.Context(),
 		cancel:  func() {},
@@ -134,7 +145,8 @@ func newTestScrapePool(t *testing.T, injectNewLoop func(options scrapeLoopOption
 		loops:             map[uint64]loop{},
 		injectTestNewLoop: injectNewLoop,
 
-		appendable:  teststorage.NewAppendable(),
+		appendable: sa.V1(), appendableV2: sa.V2(),
+
 		symbolTable: labels.NewSymbolTable(),
 		metrics:     newTestScrapeMetrics(t),
 	}
@@ -158,3 +170,66 @@ func protoMarshalDelimited(t *testing.T, mf *dto.MetricFamily) []byte {
 	buf.Write(protoBuf)
 	return buf.Bytes()
 }
+
+type selectedAppendable struct {
+	useV2 bool
+	app   compatAppendable
+}
+
+// V1 returns Appendable if V1 is selected, otherwise nil.
+func (s selectedAppendable) V1() storage.Appendable {
+	if s.useV2 {
+		return nil
+	}
+	return s.app
+}
+
+// V2 returns AppendableV2 if V2 is selected, otherwise nil.
+func (s selectedAppendable) V2() storage.AppendableV2 {
+	if !s.useV2 {
+		return nil
+	}
+	return s.app
+}
+
+// selectAppendable allows to specify which appendable callers should use when the struct
+// implements both. This is how all callers are making the decision - if one appendable is nil, they
+// take another. selectAppendable allows to inject nil to e.g. storage.AppendableV2 when appV2 is false.
+func selectAppendable(app compatAppendable, appV2 bool) selectedAppendable {
+	s := selectedAppendable{
+		app:   app,
+		useV2: appV2,
+	}
+	return s
+}
+
+func foreachAppendable(t *testing.T, f func(t *testing.T, appV2 bool)) {
+	for _, appV2 := range []bool{false, true} {
+		t.Run(fmt.Sprintf("appV2=%v", appV2), func(t *testing.T) {
+			f(t, appV2)
+		})
+	}
+}
+
+func TestSelectAppendable(t *testing.T) {
+	var i int
+	foreachAppendable(t, func(t *testing.T, appV2 bool) {
+		defer func() { i++ }()
+		switch i {
+		case 0:
+			require.False(t, appV2)
+
+			s := selectAppendable(teststorage.NewAppendable(), appV2)
+			require.NotNil(t, s.V1())
+			require.Nil(t, s.V2())
+		case 1:
+			require.True(t, appV2)
+
+			s := selectAppendable(teststorage.NewAppendable(), appV2)
+			require.Nil(t, s.V1())
+			require.NotNil(t, s.V2())
+		default:
+			t.Fatal("too many iterations")
+		}
+	})
+}

scrape/manager.go

@@ -114,7 +114,8 @@ type Manager struct {
 	opts   *Options
 	logger *slog.Logger
 
-	appendable storage.Appendable
+	appendable   storage.Appendable
+	appendableV2 storage.AppendableV2
 
 	graceShut chan struct{}
 
@@ -196,7 +197,7 @@ func (m *Manager) reload() {
 				continue
 			}
 			m.metrics.targetScrapePools.Inc()
-			sp, err := newScrapePool(scrapeConfig, m.appendable, m.offsetSeed, m.logger.With("scrape_pool", setName), m.buffers, m.opts, m.metrics)
+			sp, err := newScrapePool(scrapeConfig, m.appendable, m.appendableV2, m.offsetSeed, m.logger.With("scrape_pool", setName), m.buffers, m.opts, m.metrics)
 			if err != nil {
 				m.metrics.targetScrapePoolsFailed.Inc()
 				m.logger.Error("error creating new scrape pool", "err", err, "scrape_pool", setName)

scrape/manager_test.go

@@ -528,7 +528,7 @@ scrape_configs:
 		ch <- struct{}{}
 		return noopLoop()
 	}
-	sp := newTestScrapePool(t, newLoop)
+	sp := newTestScrapePool(t, nil, false, newLoop)
 	sp.activeTargets[1] = &Target{}
 	sp.loops[1] = noopLoop()
 	sp.config = cfg1.ScrapeConfigs[0]
@@ -684,7 +684,7 @@ scrape_configs:
 			_, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			sp := newTestScrapePool(t, newLoop)
+			sp := newTestScrapePool(t, nil, false, newLoop)
 			sp.loops[1] = noopLoop()
 			sp.config = cfg1.ScrapeConfigs[0]
 			sp.metrics = scrapeManager.metrics

scrape/metrics.go

@@ -56,6 +56,7 @@ type scrapeMetrics struct {
 	targetScrapeExemplarOutOfOrder         prometheus.Counter
 	targetScrapePoolExceededLabelLimits    prometheus.Counter
 	targetScrapeNativeHistogramBucketLimit prometheus.Counter
+	targetScrapeDuration                   prometheus.Histogram
 }
 
 func newScrapeMetrics(reg prometheus.Registerer) (*scrapeMetrics, error) {
@@ -252,6 +253,15 @@ func newScrapeMetrics(reg prometheus.Registerer) (*scrapeMetrics, error) {
 			Help: "Total number of exemplar rejected due to not being out of the expected order.",
 		},
 	)
+	sm.targetScrapeDuration = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:                            "prometheus_target_scrape_duration_seconds",
+			Help:                            "Total duration of the scrape from start to commit completion in seconds.",
+			NativeHistogramBucketFactor:     1.1,
+			NativeHistogramMaxBucketNumber:  100,
+			NativeHistogramMinResetDuration: 1 * time.Hour,
+		},
+	)
 
 	for _, collector := range []prometheus.Collector{
 		// Used by Manager.
@@ -284,6 +294,7 @@ func newScrapeMetrics(reg prometheus.Registerer) (*scrapeMetrics, error) {
 		sm.targetScrapeExemplarOutOfOrder,
 		sm.targetScrapePoolExceededLabelLimits,
 		sm.targetScrapeNativeHistogramBucketLimit,
+		sm.targetScrapeDuration,
 	} {
 		err := reg.Register(collector)
 		if err != nil {
@@ -324,6 +335,7 @@ func (sm *scrapeMetrics) Unregister() {
 	sm.reg.Unregister(sm.targetScrapeExemplarOutOfOrder)
 	sm.reg.Unregister(sm.targetScrapePoolExceededLabelLimits)
 	sm.reg.Unregister(sm.targetScrapeNativeHistogramBucketLimit)
+	sm.reg.Unregister(sm.targetScrapeDuration)
 }
 
 type TargetsGatherer interface {

scrape/scrape.go

@@ -82,11 +82,12 @@ type FailureLogger interface {
 
 // scrapePool manages scrapes for sets of targets.
 type scrapePool struct {
-	appendable storage.Appendable
-	logger     *slog.Logger
-	ctx        context.Context
-	cancel     context.CancelFunc
-	options    *Options
+	appendable   storage.Appendable
+	appendableV2 storage.AppendableV2
+	logger       *slog.Logger
+	ctx          context.Context
+	cancel       context.CancelFunc
+	options      *Options
 
 	// mtx must not be taken after targetMtx.
 	mtx    sync.Mutex
@@ -139,6 +140,7 @@ type scrapeLoopAppendAdapter interface {
 func newScrapePool(
 	cfg *config.ScrapeConfig,
 	appendable storage.Appendable,
+	appendableV2 storage.AppendableV2,
 	offsetSeed uint64,
 	logger *slog.Logger,
 	buffers *pool.Pool,
@@ -171,6 +173,7 @@ func newScrapePool(
 	ctx, cancel := context.WithCancel(context.Background())
 	sp := &scrapePool{
 		appendable:           appendable,
+		appendableV2:         appendableV2,
 		logger:               logger,
 		ctx:                  ctx,
 		cancel:               cancel,
@@ -842,11 +845,12 @@ type scrapeLoop struct {
 	scraper             scraper
 
 	// Static params per scrapePool.
-	appendable  storage.Appendable
-	buffers     *pool.Pool
-	offsetSeed  uint64
-	symbolTable *labels.SymbolTable
-	metrics     *scrapeMetrics
+	appendable   storage.Appendable
+	appendableV2 storage.AppendableV2
+	buffers      *pool.Pool
+	offsetSeed   uint64
+	symbolTable  *labels.SymbolTable
+	metrics      *scrapeMetrics
 
 	// Options from config.ScrapeConfig.
 	sampleLimit                   int
@@ -1190,11 +1194,12 @@ func newScrapeLoop(opts scrapeLoopOptions) *scrapeLoop {
 		scraper:             opts.scraper,
 
 		// Static params per scrapePool.
-		appendable:  opts.sp.appendable,
-		buffers:     opts.sp.buffers,
-		offsetSeed:  opts.sp.offsetSeed,
-		symbolTable: opts.sp.symbolTable,
-		metrics:     opts.sp.metrics,
+		appendable:   opts.sp.appendable,
+		appendableV2: opts.sp.appendableV2,
+		buffers:      opts.sp.buffers,
+		offsetSeed:   opts.sp.offsetSeed,
+		symbolTable:  opts.sp.symbolTable,
+		metrics:      opts.sp.metrics,
 
 		// config.ScrapeConfig.
 		sampleLimit: int(opts.sp.config.SampleLimit),
@@ -1303,7 +1308,9 @@ mainLoop:
 }
 
 func (sl *scrapeLoop) appender() scrapeLoopAppendAdapter {
-	// NOTE(bwplotka): Add AppenderV2 implementation, see https://github.com/prometheus/prometheus/issues/17632.
+	if sl.appendableV2 != nil {
+		return &scrapeLoopAppenderV2{scrapeLoop: sl, AppenderV2: sl.appendableV2.AppenderV2(sl.appenderCtx)}
+	}
 	return &scrapeLoopAppender{scrapeLoop: sl, Appender: sl.appendable.Appender(sl.appenderCtx)}
 }
 
@@ -1335,6 +1342,11 @@ func (sl *scrapeLoop) scrapeAndReport(last, appendTime time.Time, errc chan<- er
 			return
 		}
 		err = app.Commit()
+		if sl.reportExtraMetrics {
+			totalDuration := time.Since(start)
+			// Record total scrape duration metric.
+			sl.metrics.targetScrapeDuration.Observe(totalDuration.Seconds())
+		}
 		if err != nil {
 			sl.l.Error("Scrape commit failed", "err", err)
 		}
@@ -1632,7 +1644,7 @@ loop:
 			break
 		}
 		switch et {
-		// TODO(bwplotka): Consider changing parser to give metadata at once instead of type, help and unit in separation, ideally on `Series()/Histogram()
+		// TODO(bwplotka): Consider changing parser to give metadata at once instead of type, help and unit in separation, ideally on `Series()/Histogram()`
 		// otherwise we can expose metadata without series on metadata API.
 		case textparse.EntryType:
 			// TODO(bwplotka): Build meta entry directly instead of locking and updating the map. This will
@@ -1748,7 +1760,7 @@ loop:
 			}
 		}
 
-		sampleAdded, err = sl.checkAddError(met, err, &sampleLimitErr, &bucketLimitErr, &appErrs)
+		sampleAdded, err = sl.checkAddError(met, nil, err, &sampleLimitErr, &bucketLimitErr, &appErrs)
 		if err != nil {
 			if !errors.Is(err, storage.ErrNotFound) {
 				sl.l.Debug("Unexpected error", "series", string(met), "err", err)
@@ -1824,7 +1836,7 @@ loop:
 			if !seriesCached || lastMeta.lastIterChange == sl.cache.iter {
 				// In majority cases we can trust that the current series/histogram is matching the lastMeta and lastMFName.
 				// However, optional TYPE etc metadata and broken OM text can break this, detect those cases here.
-				// TODO(bwplotka): Consider moving this to parser as many parser users end up doing this (e.g. ST and NHCB parsing).
+				// TODO(https://github.com/prometheus/prometheus/issues/17900): Move this to text and OM parser.
 				if isSeriesPartOfFamily(lset.Get(model.MetricNameLabel), lastMFName, lastMeta.Type) {
 					if _, merr := app.UpdateMetadata(ref, lset, lastMeta.Metadata); merr != nil {
 						// No need to fail the scrape on errors appending metadata.
@@ -1866,6 +1878,7 @@ loop:
 	return total, added, seriesAdded, err
 }
 
+// TODO(https://github.com/prometheus/prometheus/issues/17900): Move this to text and OM parser.
 func isSeriesPartOfFamily(mName string, mfName []byte, typ model.MetricType) bool {
 	mfNameStr := yoloString(mfName)
 	if !strings.HasPrefix(mName, mfNameStr) { // Fast path.
@@ -1937,7 +1950,7 @@ func isSeriesPartOfFamily(mName string, mfName []byte, typ model.MetricType) boo
 // during normal operation (e.g., accidental cardinality explosion, sudden traffic spikes).
 // Current case ordering prevents exercising other cases when limits are exceeded.
 // Remaining error cases typically occur only a few times, often during initial setup.
-func (sl *scrapeLoop) checkAddError(met []byte, err error, sampleLimitErr, bucketLimitErr *error, appErrs *appendErrors) (sampleAdded bool, _ error) {
+func (sl *scrapeLoop) checkAddError(met []byte, exemplars []exemplar.Exemplar, err error, sampleLimitErr, bucketLimitErr *error, appErrs *appendErrors) (sampleAdded bool, _ error) {
 	switch {
 	case err == nil:
 		return true, nil
@@ -1969,6 +1982,26 @@ func (sl *scrapeLoop) checkAddError(met []byte, err error, sampleLimitErr, bucke
 	case errors.Is(err, storage.ErrNotFound):
 		return false, storage.ErrNotFound
 	default:
+		// If nothing from the above, check for partial errors. Do this here to not alloc the pErr on a hot path.
+		var pErr *storage.AppendPartialError
+		if errors.As(err, &pErr) {
+			outOfOrderExemplars := 0
+			for _, e := range pErr.ExemplarErrors {
+				if errors.Is(e, storage.ErrOutOfOrderExemplar) {
+					outOfOrderExemplars++
+				}
+				// Since exemplar storage is still experimental, we don't fail or check other errors.
+				// Debug log is emitted in TSDB already.
+			}
+			if outOfOrderExemplars > 0 && outOfOrderExemplars == len(exemplars) {
+				// Only report out of order exemplars if all are out of order, otherwise this was a partial update
+				// to some existing set of exemplars.
+				appErrs.numExemplarOutOfOrder += outOfOrderExemplars
+				sl.l.Debug("Out of order exemplars", "count", outOfOrderExemplars, "latest", fmt.Sprintf("%+v", exemplars[len(exemplars)-1]))
+				sl.metrics.targetScrapeExemplarOutOfOrder.Add(float64(outOfOrderExemplars))
+			}
+			return true, nil
+		}
 		return false, err
 	}
 }

scrape/scrape_append_v2.go

@@ -0,0 +1,416 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scrape
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"slices"
+	"time"
+
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/prometheus/model/exemplar"
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/textparse"
+	"github.com/prometheus/prometheus/model/timestamp"
+	"github.com/prometheus/prometheus/model/value"
+	"github.com/prometheus/prometheus/storage"
+)
+
+// appenderWithLimits returns an appender with additional validation.
+func appenderV2WithLimits(app storage.AppenderV2, sampleLimit, bucketLimit int, maxSchema int32) storage.AppenderV2 {
+	app = &timeLimitAppenderV2{
+		AppenderV2: app,
+		maxTime:    timestamp.FromTime(time.Now().Add(maxAheadTime)),
+	}
+
+	// The sampleLimit is applied after metrics are potentially dropped via relabeling.
+	if sampleLimit > 0 {
+		app = &limitAppenderV2{
+			AppenderV2: app,
+			limit:      sampleLimit,
+		}
+	}
+
+	if bucketLimit > 0 {
+		app = &bucketLimitAppenderV2{
+			AppenderV2: app,
+			limit:      bucketLimit,
+		}
+	}
+
+	if maxSchema < histogram.ExponentialSchemaMax {
+		app = &maxSchemaAppenderV2{
+			AppenderV2: app,
+			maxSchema:  maxSchema,
+		}
+	}
+
+	return app
+}
+
+func (sl *scrapeLoop) updateStaleMarkersV2(app storage.AppenderV2, defTime int64) (err error) {
+	sl.cache.forEachStale(func(ref storage.SeriesRef, lset labels.Labels) bool {
+		// Series no longer exposed, mark it stale.
+		_, err = app.Append(ref, lset, 0, defTime, math.Float64frombits(value.StaleNaN), nil, nil, storage.AOptions{RejectOutOfOrder: true})
+		switch {
+		case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrDuplicateSampleForTimestamp):
+			// Do not count these in logging, as this is expected if a target
+			// goes away and comes back again with a new scrape loop.
+			err = nil
+		}
+		return err == nil
+	})
+	return err
+}
+
+type scrapeLoopAppenderV2 struct {
+	*scrapeLoop
+
+	storage.AppenderV2
+}
+
+var _ scrapeLoopAppendAdapter = &scrapeLoopAppenderV2{}
+
+func (sl *scrapeLoopAppenderV2) append(b []byte, contentType string, ts time.Time) (total, added, seriesAdded int, err error) {
+	defTime := timestamp.FromTime(ts)
+
+	if len(b) == 0 {
+		// Empty scrape. Just update the stale makers and swap the cache (but don't flush it).
+		err = sl.updateStaleMarkersV2(sl.AppenderV2, defTime)
+		sl.cache.iterDone(false)
+		return total, added, seriesAdded, err
+	}
+
+	p, err := textparse.New(b, contentType, sl.symbolTable, textparse.ParserOptions{
+		EnableTypeAndUnitLabels:                 sl.enableTypeAndUnitLabels,
+		IgnoreNativeHistograms:                  !sl.enableNativeHistogramScraping,
+		ConvertClassicHistogramsToNHCB:          sl.convertClassicHistToNHCB,
+		KeepClassicOnClassicAndNativeHistograms: sl.alwaysScrapeClassicHist,
+		OpenMetricsSkipSTSeries:                 sl.enableSTZeroIngestion,
+		FallbackContentType:                     sl.fallbackScrapeProtocol,
+	})
+	if p == nil {
+		sl.l.Error(
+			"Failed to determine correct type of scrape target.",
+			"content_type", contentType,
+			"fallback_media_type", sl.fallbackScrapeProtocol,
+			"err", err,
+		)
+		return total, added, seriesAdded, err
+	}
+	if err != nil {
+		sl.l.Debug(
+			"Invalid content type on scrape, using fallback setting.",
+			"content_type", contentType,
+			"fallback_media_type", sl.fallbackScrapeProtocol,
+			"err", err,
+		)
+	}
+	var (
+		appErrs        = appendErrors{}
+		sampleLimitErr error
+		bucketLimitErr error
+		lset           labels.Labels     // Escapes to heap so hoisted out of loop.
+		e              exemplar.Exemplar // Escapes to heap so hoisted out of loop.
+		lastMeta       *metaEntry
+		lastMFName     []byte
+	)
+
+	exemplars := make([]exemplar.Exemplar, 0, 1)
+
+	// Take an appender with limits.
+	app := appenderV2WithLimits(sl.AppenderV2, sl.sampleLimit, sl.bucketLimit, sl.maxSchema)
+
+	defer func() {
+		if err != nil {
+			return
+		}
+		// Flush and swap the cache as the scrape was non-empty.
+		sl.cache.iterDone(true)
+	}()
+
+loop:
+	for {
+		var (
+			et                       textparse.Entry
+			sampleAdded, isHistogram bool
+			met                      []byte
+			parsedTimestamp          *int64
+			val                      float64
+			h                        *histogram.Histogram
+			fh                       *histogram.FloatHistogram
+		)
+		if et, err = p.Next(); err != nil {
+			if errors.Is(err, io.EOF) {
+				err = nil
+			}
+			break
+		}
+		switch et {
+		// TODO(bwplotka): Consider changing parser to give metadata at once instead of type, help and unit in separation, ideally on `Series()/Histogram()
+		// otherwise we can expose metadata without series on metadata API.
+		case textparse.EntryType:
+			// TODO(bwplotka): Build meta entry directly instead of locking and updating the map. This will
+			// allow to properly update metadata when e.g unit was added, then removed;
+			lastMFName, lastMeta = sl.cache.setType(p.Type())
+			continue
+		case textparse.EntryHelp:
+			lastMFName, lastMeta = sl.cache.setHelp(p.Help())
+			continue
+		case textparse.EntryUnit:
+			lastMFName, lastMeta = sl.cache.setUnit(p.Unit())
+			continue
+		case textparse.EntryComment:
+			continue
+		case textparse.EntryHistogram:
+			isHistogram = true
+		default:
+		}
+		total++
+
+		t := defTime
+		if isHistogram {
+			met, parsedTimestamp, h, fh = p.Histogram()
+		} else {
+			met, parsedTimestamp, val = p.Series()
+		}
+		if !sl.honorTimestamps {
+			parsedTimestamp = nil
+		}
+		if parsedTimestamp != nil {
+			t = *parsedTimestamp
+		}
+
+		if sl.cache.getDropped(met) {
+			continue
+		}
+		ce, seriesCached, seriesAlreadyScraped := sl.cache.get(met)
+		var (
+			ref  storage.SeriesRef
+			hash uint64
+		)
+
+		if seriesCached {
+			ref = ce.ref
+			lset = ce.lset
+			hash = ce.hash
+		} else {
+			p.Labels(&lset)
+			hash = lset.Hash()
+
+			// Hash label set as it is seen local to the target. Then add target labels
+			// and relabeling and store the final label set.
+			lset = sl.sampleMutator(lset)
+
+			// The label set may be set to empty to indicate dropping.
+			if lset.IsEmpty() {
+				sl.cache.addDropped(met)
+				continue
+			}
+
+			if !lset.Has(model.MetricNameLabel) {
+				err = errNameLabelMandatory
+				break loop
+			}
+			if !lset.IsValid(sl.validationScheme) {
+				err = fmt.Errorf("invalid metric name or label names: %s", lset.String())
+				break loop
+			}
+
+			// If any label limits is exceeded the scrape should fail.
+			if err = verifyLabelLimits(lset, sl.labelLimits); err != nil {
+				sl.metrics.targetScrapePoolExceededLabelLimits.Inc()
+				break loop
+			}
+		}
+
+		exemplars = exemplars[:0] // Reset and reuse the exemplar slice.
+
+		if seriesAlreadyScraped && parsedTimestamp == nil {
+			err = storage.ErrDuplicateSampleForTimestamp
+		} else {
+			// Double check we don't append float 0 for
+			// histogram case where parser returns bad data.
+			// This can only happen when parser has a bug.
+			if isHistogram && h == nil && fh == nil {
+				err = fmt.Errorf("parser returned nil histogram/float histogram for a histogram entry type for %v series; parser bug; aborting", lset.String())
+				break loop
+			}
+
+			st := int64(0)
+			if sl.enableSTZeroIngestion {
+				// p.StartTimestamp() tend to be expensive (e.g. OM1). Do it only if we care.
+				st = p.StartTimestamp()
+			}
+
+			for hasExemplar := p.Exemplar(&e); hasExemplar; hasExemplar = p.Exemplar(&e) {
+				if !e.HasTs {
+					if isHistogram {
+						// We drop exemplars for native histograms if they don't have a timestamp.
+						// Missing timestamps are deliberately not supported as we want to start
+						// enforcing timestamps for exemplars as otherwise proper deduplication
+						// is inefficient and purely based on heuristics: we cannot distinguish
+						// between repeated exemplars and new instances with the same values.
+						// This is done silently without logs as it is not an error but out of spec.
+						// This does not affect classic histograms so that behaviour is unchanged.
+						e = exemplar.Exemplar{} // Reset for the next fetch.
+						continue
+					}
+					e.Ts = t
+				}
+				exemplars = append(exemplars, e)
+				e = exemplar.Exemplar{} // Reset for the next fetch.
+			}
+
+			// Prepare append call.
+			appOpts := storage.AOptions{}
+			if len(exemplars) > 0 {
+				// Sort so that checking for duplicates / out of order is more efficient during validation.
+				slices.SortFunc(exemplars, exemplar.Compare)
+				appOpts.Exemplars = exemplars
+			}
+
+			// Metadata path mimicks the scrape appender V1 flow. Once we remove v2
+			// flow we should rename "appendMetadataToWAL" flag to "passMetadata" because for v2 flow
+			// the metadata storage detail is behind the appendableV2 contract. V2 also means we always pass the metadata,
+			// we don't check if it changed (that code can be removed).
+			//
+			// Long term, we should always attach the metadata without any flag. Unfortunately because of the limitation
+			// of the TEXT and OpenMetrics 1.0 (hopefully fixed in OpenMetrics 2.0) there are edge cases around unknown
+			// metadata + suffixes that is expensive (isSeriesPartOfFamily) or in some cases impossible to detect. For this
+			// reason metadata (appendMetadataToWAL=true) appender V2 flow scrape might taking ~3% more CPU in our benchmarks.
+			//
+			// TODO(https://github.com/prometheus/prometheus/issues/17900): Optimize this, notably move this check to parsers that require this (ensuring parser
+			// interface always yields correct metadata), deliver OpenMetrics 2.0 that removes suffixes.
+			if sl.appendMetadataToWAL && lastMeta != nil {
+				// In majority cases we can trust that the current series/histogram is matching the lastMeta and lastMFName.
+				// However, optional TYPE, etc metadata and broken OM text can break this, detect those cases here.
+				if !isSeriesPartOfFamily(lset.Get(model.MetricNameLabel), lastMFName, lastMeta.Type) {
+					lastMeta = nil // Don't pass knowingly broken metadata, now, nor on the next line.
+				}
+				if lastMeta != nil {
+					// Metric family name has the same source as metadata.
+					appOpts.MetricFamilyName = yoloString(lastMFName)
+					appOpts.Metadata = lastMeta.Metadata
+				}
+			}
+
+			// Append sample to the storage.
+			ref, err = app.Append(ref, lset, st, t, val, h, fh, appOpts)
+		}
+		sampleAdded, err = sl.checkAddError(met, exemplars, err, &sampleLimitErr, &bucketLimitErr, &appErrs)
+		if err != nil {
+			if !errors.Is(err, storage.ErrNotFound) {
+				sl.l.Debug("Unexpected error", "series", string(met), "err", err)
+			}
+			break loop
+		}
+		if (parsedTimestamp == nil || sl.trackTimestampsStaleness) && ce != nil {
+			sl.cache.trackStaleness(ce.ref, ce)
+		}
+
+		// If series wasn't cached (is new, not seen on previous scrape) we need to add it to the scrape cache.
+		// But we only do this for series that were appended to TSDB without errors.
+		// If a series was new, but we didn't append it due to sample_limit or other errors then we don't need
+		// it in the scrape cache because we don't need to emit StaleNaNs for it when it disappears.
+		if !seriesCached && sampleAdded {
+			ce = sl.cache.addRef(met, ref, lset, hash)
+			if ce != nil && (parsedTimestamp == nil || sl.trackTimestampsStaleness) {
+				// Bypass staleness logic if there is an explicit timestamp.
+				// But make sure we only do this if we have a cache entry (ce) for our series.
+				sl.cache.trackStaleness(ref, ce)
+			}
+			if sampleLimitErr == nil && bucketLimitErr == nil {
+				seriesAdded++
+			}
+		}
+
+		// Increment added even if there's an error so we correctly report the
+		// number of samples remaining after relabeling.
+		// We still report duplicated samples here since this number should be the exact number
+		// of time series exposed on a scrape after relabelling.
+		added++
+	}
+	if sampleLimitErr != nil {
+		if err == nil {
+			err = sampleLimitErr
+		}
+		// We only want to increment this once per scrape, so this is Inc'd outside the loop.
+		sl.metrics.targetScrapeSampleLimit.Inc()
+	}
+	if bucketLimitErr != nil {
+		if err == nil {
+			err = bucketLimitErr // If sample limit is hit, that error takes precedence.
+		}
+		// We only want to increment this once per scrape, so this is Inc'd outside the loop.
+		sl.metrics.targetScrapeNativeHistogramBucketLimit.Inc()
+	}
+	if appErrs.numOutOfOrder > 0 {
+		sl.l.Warn("Error on ingesting out-of-order samples", "num_dropped", appErrs.numOutOfOrder)
+	}
+	if appErrs.numDuplicates > 0 {
+		sl.l.Warn("Error on ingesting samples with different value but same timestamp", "num_dropped", appErrs.numDuplicates)
+	}
+	if appErrs.numOutOfBounds > 0 {
+		sl.l.Warn("Error on ingesting samples that are too old or are too far into the future", "num_dropped", appErrs.numOutOfBounds)
+	}
+	if appErrs.numExemplarOutOfOrder > 0 {
+		sl.l.Warn("Error on ingesting out-of-order exemplars", "num_dropped", appErrs.numExemplarOutOfOrder)
+	}
+	if err == nil {
+		err = sl.updateStaleMarkersV2(app, defTime)
+	}
+	return total, added, seriesAdded, err
+}
+
+func (sl *scrapeLoopAppenderV2) addReportSample(s reportSample, t int64, v float64, b *labels.Builder, rejectOOO bool) (err error) {
+	ce, ok, _ := sl.cache.get(s.name)
+	var ref storage.SeriesRef
+	var lset labels.Labels
+	if ok {
+		ref = ce.ref
+		lset = ce.lset
+	} else {
+		// The constants are suffixed with the invalid \xff unicode rune to avoid collisions
+		// with scraped metrics in the cache.
+		// We have to drop it when building the actual metric.
+		b.Reset(labels.EmptyLabels())
+		b.Set(model.MetricNameLabel, string(s.name[:len(s.name)-1]))
+		lset = sl.reportSampleMutator(b.Labels())
+	}
+
+	ref, err = sl.Append(ref, lset, 0, t, v, nil, nil, storage.AOptions{
+		MetricFamilyName: yoloString(s.name),
+		Metadata:         s.Metadata,
+		RejectOutOfOrder: rejectOOO,
+	})
+	switch {
+	case err == nil:
+		if !ok {
+			sl.cache.addRef(s.name, ref, lset, lset.Hash())
+		}
+		return nil
+	case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrDuplicateSampleForTimestamp):
+		// Do not log here, as this is expected if a target goes away and comes back
+		// again with a new scrape loop.
+		return nil
+	default:
+		return err
+	}
+}

scrape/target.go

@@ -454,6 +454,105 @@ func (app *maxSchemaAppender) AppendHistogram(ref storage.SeriesRef, lset labels
 	return ref, nil
 }
 
+// limitAppender limits the number of total appended samples in a batch.
+type limitAppenderV2 struct {
+	storage.AppenderV2
+
+	limit int
+	i     int
+}
+
+func (app *limitAppenderV2) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) {
+	// Bypass sample_limit checks only if we have a staleness marker for a known series (ref value is non-zero).
+	// This ensures that if a series is already in TSDB then we always write the marker.
+	if ref == 0 || !value.IsStaleNaN(v) {
+		app.i++
+		if app.i > app.limit {
+			return 0, errSampleLimit
+		}
+	}
+	return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts)
+}
+
+type timeLimitAppenderV2 struct {
+	storage.AppenderV2
+
+	maxTime int64
+}
+
+func (app *timeLimitAppenderV2) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) {
+	if t > app.maxTime {
+		return 0, storage.ErrOutOfBounds
+	}
+
+	return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts)
+}
+
+// bucketLimitAppender limits the number of total appended samples in a batch.
+type bucketLimitAppenderV2 struct {
+	storage.AppenderV2
+
+	limit int
+}
+
+func (app *bucketLimitAppenderV2) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (_ storage.SeriesRef, err error) {
+	if h != nil {
+		// Return with an early error if the histogram has too many buckets and the
+		// schema is not exponential, in which case we can't reduce the resolution.
+		if len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit && !histogram.IsExponentialSchema(h.Schema) {
+			return 0, errBucketLimit
+		}
+		for len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit {
+			if h.Schema <= histogram.ExponentialSchemaMin {
+				return 0, errBucketLimit
+			}
+			if err = h.ReduceResolution(h.Schema - 1); err != nil {
+				return 0, err
+			}
+		}
+	}
+	if fh != nil {
+		// Return with an early error if the histogram has too many buckets and the
+		// schema is not exponential, in which case we can't reduce the resolution.
+		if len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit && !histogram.IsExponentialSchema(fh.Schema) {
+			return 0, errBucketLimit
+		}
+		for len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit {
+			if fh.Schema <= histogram.ExponentialSchemaMin {
+				return 0, errBucketLimit
+			}
+			if err = fh.ReduceResolution(fh.Schema - 1); err != nil {
+				return 0, err
+			}
+		}
+	}
+	return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts)
+}
+
+type maxSchemaAppenderV2 struct {
+	storage.AppenderV2
+
+	maxSchema int32
+}
+
+func (app *maxSchemaAppenderV2) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (_ storage.SeriesRef, err error) {
+	if h != nil {
+		if histogram.IsExponentialSchemaReserved(h.Schema) && h.Schema > app.maxSchema {
+			if err = h.ReduceResolution(app.maxSchema); err != nil {
+				return 0, err
+			}
+		}
+	}
+	if fh != nil {
+		if histogram.IsExponentialSchemaReserved(fh.Schema) && fh.Schema > app.maxSchema {
+			if err = fh.ReduceResolution(app.maxSchema); err != nil {
+				return 0, err
+			}
+		}
+	}
+	return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts)
+}
+
 // PopulateDiscoveredLabels sets base labels on lb from target and group labels and scrape configuration, before relabeling.
 func PopulateDiscoveredLabels(lb *labels.Builder, cfg *config.ScrapeConfig, tLabels, tgLabels model.LabelSet) {
 	lb.Reset(labels.EmptyLabels())

scrape/target_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/timestamp"
+	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/util/teststorage"
 )
 
@@ -610,37 +611,65 @@ func TestBucketLimitAppender(t *testing.T) {
 		},
 	}
 
-	appTest := teststorage.NewAppendable()
-
 	for _, c := range cases {
 		for _, floatHisto := range []bool{true, false} {
 			t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) {
-				app := &bucketLimitAppender{Appender: appTest.Appender(t.Context()), limit: c.limit}
-				ts := int64(10 * time.Minute / time.Millisecond)
-				lbls := labels.FromStrings("__name__", "sparse_histogram_series")
-				var err error
-				if floatHisto {
-					fh := c.h.Copy().ToFloat(nil)
-					_, err = app.AppendHistogram(0, lbls, ts, nil, fh)
-					if c.expectError {
-						require.Error(t, err)
+				t.Run("appV2=false", func(t *testing.T) {
+					app := &bucketLimitAppender{Appender: teststorage.NewAppendable().Appender(t.Context()), limit: c.limit}
+					ts := int64(10 * time.Minute / time.Millisecond)
+					lbls := labels.FromStrings("__name__", "sparse_histogram_series")
+					var err error
+					if floatHisto {
+						fh := c.h.Copy().ToFloat(nil)
+						_, err = app.AppendHistogram(0, lbls, ts, nil, fh)
+						if c.expectError {
+							require.Error(t, err)
+						} else {
+							require.Equal(t, c.expectSchema, fh.Schema)
+							require.Equal(t, c.expectBucketCount, len(fh.NegativeBuckets)+len(fh.PositiveBuckets))
+							require.NoError(t, err)
+						}
 					} else {
-						require.Equal(t, c.expectSchema, fh.Schema)
-						require.Equal(t, c.expectBucketCount, len(fh.NegativeBuckets)+len(fh.PositiveBuckets))
-						require.NoError(t, err)
+						h := c.h.Copy()
+						_, err = app.AppendHistogram(0, lbls, ts, h, nil)
+						if c.expectError {
+							require.Error(t, err)
+						} else {
+							require.Equal(t, c.expectSchema, h.Schema)
+							require.Equal(t, c.expectBucketCount, len(h.NegativeBuckets)+len(h.PositiveBuckets))
+							require.NoError(t, err)
+						}
 					}
-				} else {
-					h := c.h.Copy()
-					_, err = app.AppendHistogram(0, lbls, ts, h, nil)
-					if c.expectError {
-						require.Error(t, err)
+					require.NoError(t, app.Commit())
+				})
+				t.Run("appV2=true", func(t *testing.T) {
+					app := &bucketLimitAppenderV2{AppenderV2: teststorage.NewAppendable().AppenderV2(t.Context()), limit: c.limit}
+					ts := int64(10 * time.Minute / time.Millisecond)
+					lbls := labels.FromStrings("__name__", "sparse_histogram_series")
+					var err error
+					if floatHisto {
+						fh := c.h.Copy().ToFloat(nil)
+						_, err = app.Append(0, lbls, 0, ts, 0, nil, fh, storage.AOptions{})
+						if c.expectError {
+							require.Error(t, err)
+						} else {
+							require.Equal(t, c.expectSchema, fh.Schema)
+							require.Equal(t, c.expectBucketCount, len(fh.NegativeBuckets)+len(fh.PositiveBuckets))
+							require.NoError(t, err)
+						}
 					} else {
-						require.Equal(t, c.expectSchema, h.Schema)
-						require.Equal(t, c.expectBucketCount, len(h.NegativeBuckets)+len(h.PositiveBuckets))
-						require.NoError(t, err)
+						h := c.h.Copy()
+						_, err = app.Append(0, lbls, 0, ts, 0, h, nil, storage.AOptions{})
+						if c.expectError {
+							require.Error(t, err)
+						} else {
+							require.Equal(t, c.expectSchema, h.Schema)
+							require.Equal(t, c.expectBucketCount, len(h.NegativeBuckets)+len(h.PositiveBuckets))
+							require.NoError(t, err)
+						}
 					}
-				}
-				require.NoError(t, app.Commit())
+					require.NoError(t, app.Commit())
+				})
 			})
 		}
 	}
@@ -696,27 +725,45 @@ func TestMaxSchemaAppender(t *testing.T) {
 		},
 	}
 
-	appTest := teststorage.NewAppendable()
-
 	for _, c := range cases {
 		for _, floatHisto := range []bool{true, false} {
 			t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) {
-				app := &maxSchemaAppender{Appender: appTest.Appender(t.Context()), maxSchema: c.maxSchema}
-				ts := int64(10 * time.Minute / time.Millisecond)
-				lbls := labels.FromStrings("__name__", "sparse_histogram_series")
-				var err error
-				if floatHisto {
-					fh := c.h.Copy().ToFloat(nil)
-					_, err = app.AppendHistogram(0, lbls, ts, nil, fh)
-					require.Equal(t, c.expectSchema, fh.Schema)
-					require.NoError(t, err)
-				} else {
-					h := c.h.Copy()
-					_, err = app.AppendHistogram(0, lbls, ts, h, nil)
-					require.Equal(t, c.expectSchema, h.Schema)
-					require.NoError(t, err)
-				}
-				require.NoError(t, app.Commit())
+				t.Run("appV2=false", func(t *testing.T) {
+					app := &maxSchemaAppender{Appender: teststorage.NewAppendable().Appender(t.Context()), maxSchema: c.maxSchema}
+					ts := int64(10 * time.Minute / time.Millisecond)
+					lbls := labels.FromStrings("__name__", "sparse_histogram_series")
+					var err error
+					if floatHisto {
+						fh := c.h.Copy().ToFloat(nil)
+						_, err = app.AppendHistogram(0, lbls, ts, nil, fh)
+						require.Equal(t, c.expectSchema, fh.Schema)
+						require.NoError(t, err)
+					} else {
+						h := c.h.Copy()
+						_, err = app.AppendHistogram(0, lbls, ts, h, nil)
+						require.Equal(t, c.expectSchema, h.Schema)
+						require.NoError(t, err)
+					}
+					require.NoError(t, app.Commit())
+				})
+				t.Run("appV2=true", func(t *testing.T) {
+					app := &maxSchemaAppenderV2{AppenderV2: teststorage.NewAppendable().AppenderV2(t.Context()), maxSchema: c.maxSchema}
+					ts := int64(10 * time.Minute / time.Millisecond)
+					lbls := labels.FromStrings("__name__", "sparse_histogram_series")
+					var err error
+					if floatHisto {
+						fh := c.h.Copy().ToFloat(nil)
+						_, err = app.Append(0, lbls, 0, ts, 0, nil, fh, storage.AOptions{})
+						require.Equal(t, c.expectSchema, fh.Schema)
+						require.NoError(t, err)
+					} else {
+						h := c.h.Copy()
+						_, err = app.Append(0, lbls, 0, ts, 0, h, nil, storage.AOptions{})
+						require.Equal(t, c.expectSchema, h.Schema)
+						require.NoError(t, err)
+					}
+					require.NoError(t, app.Commit())
+				})
 			})
 		}
 	}
@@ -724,32 +771,65 @@ func TestMaxSchemaAppender(t *testing.T) {
 
 // Test sample_limit when a scrape contains Native Histograms.
 func TestAppendWithSampleLimitAndNativeHistogram(t *testing.T) {
-	appTest := teststorage.NewAppendable()
-
 	now := time.Now()
-	app := appenderWithLimits(appTest.Appender(t.Context()), 2, 0, histogram.ExponentialSchemaMax)
+	t.Run("appV2=false", func(t *testing.T) {
+		app := appenderWithLimits(teststorage.NewAppendable().Appender(t.Context()), 2, 0, histogram.ExponentialSchemaMax)
 
-	// sample_limit is set to 2, so first two scrapes should work
-	_, err := app.Append(0, labels.FromStrings(model.MetricNameLabel, "foo"), timestamp.FromTime(now), 1)
-	require.NoError(t, err)
+		// sample_limit is set to 2, so first two scrapes should work
+		_, err := app.Append(0, labels.FromStrings(model.MetricNameLabel, "foo"), timestamp.FromTime(now), 1)
+		require.NoError(t, err)
 
-	// Second sample, should be ok.
-	_, err = app.AppendHistogram(
-		0,
-		labels.FromStrings(model.MetricNameLabel, "my_histogram1"),
-		timestamp.FromTime(now),
-		&histogram.Histogram{},
-		nil,
-	)
-	require.NoError(t, err)
+		// Second sample, should be ok.
+		_, err = app.AppendHistogram(
+			0,
+			labels.FromStrings(model.MetricNameLabel, "my_histogram1"),
+			timestamp.FromTime(now),
+			&histogram.Histogram{},
+			nil,
+		)
+		require.NoError(t, err)
 
-	// This is third sample with sample_limit=2, it should trigger errSampleLimit.
-	_, err = app.AppendHistogram(
-		0,
-		labels.FromStrings(model.MetricNameLabel, "my_histogram2"),
-		timestamp.FromTime(now),
-		&histogram.Histogram{},
-		nil,
-	)
-	require.ErrorIs(t, err, errSampleLimit)
+		// This is third sample with sample_limit=2, it should trigger errSampleLimit.
+		_, err = app.AppendHistogram(
+			0,
+			labels.FromStrings(model.MetricNameLabel, "my_histogram2"),
+			timestamp.FromTime(now),
+			&histogram.Histogram{},
+			nil,
+		)
+		require.ErrorIs(t, err, errSampleLimit)
+	})
+	t.Run("appV2=true", func(t *testing.T) {
+		app := appenderV2WithLimits(teststorage.NewAppendable().AppenderV2(t.Context()), 2, 0, histogram.ExponentialSchemaMax)
+
+		// sample_limit is set to 2, so first two scrapes should work
+		_, err := app.Append(0, labels.FromStrings(model.MetricNameLabel, "foo"), 0, timestamp.FromTime(now), 1, nil, nil, storage.AOptions{})
+		require.NoError(t, err)
+
+		// Second sample, should be ok.
+		_, err = app.Append(
+			0,
+			labels.FromStrings(model.MetricNameLabel, "my_histogram1"),
+			0,
+			timestamp.FromTime(now),
+			0,
+			&histogram.Histogram{},
+			nil,
+			storage.AOptions{},
+		)
+		require.NoError(t, err)
+
+		// This is third sample with sample_limit=2, it should trigger errSampleLimit.
+		_, err = app.Append(
+			0,
+			labels.FromStrings(model.MetricNameLabel, "my_histogram2"),
+			0,
+			timestamp.FromTime(now),
+			0,
+			&histogram.Histogram{},
+			nil,
+			storage.AOptions{},
+		)
+		require.ErrorIs(t, err, errSampleLimit)
+	})
 }

scripts/check-go-mod-version.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
 
-readarray -t mod_files < <(find . -type f -name go.mod)
+readarray -t mod_files < <(git ls-files go.mod go.work '*/go.mod' || find . -type f -name go.mod -or -name go.work)
 
 echo "Checking files ${mod_files[@]}"
 
 matches=$(awk '$1 == "go" {print $2}' "${mod_files[@]}" | sort -u | wc -l)
 
 if [[ "${matches}" -ne 1 ]]; then
-  echo 'Not all go.mod files have matching go versions'
+  echo 'Not all go.mod/go.work files have matching go versions'
   exit 1
 fi

storage/buffer.go

@@ -119,13 +119,16 @@ func (b *BufferedSeriesIterator) Next() chunkenc.ValueType {
 		return chunkenc.ValNone
 	case chunkenc.ValFloat:
 		t, f := b.it.At()
-		b.buf.addF(fSample{t: t, f: f})
+		st := b.it.AtST()
+		b.buf.addF(fSample{st: st, t: t, f: f})
 	case chunkenc.ValHistogram:
 		t, h := b.it.AtHistogram(&b.hReader)
-		b.buf.addH(hSample{t: t, h: h})
+		st := b.it.AtST()
+		b.buf.addH(hSample{st: st, t: t, h: h})
 	case chunkenc.ValFloatHistogram:
 		t, fh := b.it.AtFloatHistogram(&b.fhReader)
-		b.buf.addFH(fhSample{t: t, fh: fh})
+		st := b.it.AtST()
+		b.buf.addFH(fhSample{st: st, t: t, fh: fh})
 	default:
 		panic(fmt.Errorf("BufferedSeriesIterator: unknown value type %v", b.valueType))
 	}
@@ -157,20 +160,29 @@ func (b *BufferedSeriesIterator) AtT() int64 {
 	return b.it.AtT()
 }
 
+// AtST returns the current sample's start timestamp of the iterator.
+func (b *BufferedSeriesIterator) AtST() int64 {
+	return b.it.AtST()
+}
+
 // Err returns the last encountered error.
 func (b *BufferedSeriesIterator) Err() error {
 	return b.it.Err()
 }
 
 type fSample struct {
-	t int64
-	f float64
+	st, t int64
+	f     float64
 }
 
 func (s fSample) T() int64 {
 	return s.t
 }
 
+func (s fSample) ST() int64 {
+	return s.st
+}
+
 func (s fSample) F() float64 {
 	return s.f
 }
@@ -192,14 +204,18 @@ func (s fSample) Copy() chunks.Sample {
 }
 
 type hSample struct {
-	t int64
-	h *histogram.Histogram
+	st, t int64
+	h     *histogram.Histogram
 }
 
 func (s hSample) T() int64 {
 	return s.t
 }
 
+func (s hSample) ST() int64 {
+	return s.st
+}
+
 func (hSample) F() float64 {
 	panic("F() called for hSample")
 }
@@ -217,18 +233,22 @@ func (hSample) Type() chunkenc.ValueType {
 }
 
 func (s hSample) Copy() chunks.Sample {
-	return hSample{t: s.t, h: s.h.Copy()}
+	return hSample{st: s.st, t: s.t, h: s.h.Copy()}
 }
 
 type fhSample struct {
-	t  int64
-	fh *histogram.FloatHistogram
+	st, t int64
+	fh    *histogram.FloatHistogram
 }
 
 func (s fhSample) T() int64 {
 	return s.t
 }
 
+func (s fhSample) ST() int64 {
+	return s.st
+}
+
 func (fhSample) F() float64 {
 	panic("F() called for fhSample")
 }
@@ -246,7 +266,7 @@ func (fhSample) Type() chunkenc.ValueType {
 }
 
 func (s fhSample) Copy() chunks.Sample {
-	return fhSample{t: s.t, fh: s.fh.Copy()}
+	return fhSample{st: s.st, t: s.t, fh: s.fh.Copy()}
 }
 
 type sampleRing struct {
@@ -329,6 +349,7 @@ func (r *sampleRing) iterator() *SampleRingIterator {
 type SampleRingIterator struct {
 	r  *sampleRing
 	i  int
+	st int64
 	t  int64
 	f  float64
 	h  *histogram.Histogram
@@ -350,21 +371,25 @@ func (it *SampleRingIterator) Next() chunkenc.ValueType {
 	switch it.r.bufInUse {
 	case fBuf:
 		s := it.r.atF(it.i)
+		it.st = s.st
 		it.t = s.t
 		it.f = s.f
 		return chunkenc.ValFloat
 	case hBuf:
 		s := it.r.atH(it.i)
+		it.st = s.st
 		it.t = s.t
 		it.h = s.h
 		return chunkenc.ValHistogram
 	case fhBuf:
 		s := it.r.atFH(it.i)
+		it.st = s.st
 		it.t = s.t
 		it.fh = s.fh
 		return chunkenc.ValFloatHistogram
 	}
 	s := it.r.at(it.i)
+	it.st = s.ST()
 	it.t = s.T()
 	switch s.Type() {
 	case chunkenc.ValHistogram:
@@ -410,6 +435,10 @@ func (it *SampleRingIterator) AtT() int64 {
 	return it.t
 }
 
+func (it *SampleRingIterator) AtST() int64 {
+	return it.st
+}
+
 func (r *sampleRing) at(i int) chunks.Sample {
 	j := (r.f + i) % len(r.iBuf)
 	return r.iBuf[j]
@@ -651,6 +680,7 @@ func addH(s hSample, buf []hSample, r *sampleRing) []hSample {
 	}
 
 	buf[r.i].t = s.t
+	buf[r.i].st = s.st
 	if buf[r.i].h == nil {
 		buf[r.i].h = s.h.Copy()
 	} else {
@@ -695,6 +725,7 @@ func addFH(s fhSample, buf []fhSample, r *sampleRing) []fhSample {
 	}
 
 	buf[r.i].t = s.t
+	buf[r.i].st = s.st
 	if buf[r.i].fh == nil {
 		buf[r.i].fh = s.fh.Copy()
 	} else {

storage/buffer_test.go

@@ -61,10 +61,9 @@ func TestSampleRing(t *testing.T) {
 
 		input := []fSample{}
 		for _, t := range c.input {
-			input = append(input, fSample{
-				t: t,
-				f: float64(rand.Intn(100)),
-			})
+			// Randomize start timestamp to make sure it does not affect the
+			// outcome.
+			input = append(input, fSample{st: rand.Int63(), t: t, f: float64(rand.Intn(100))})
 		}
 
 		for i, s := range input {
@@ -90,6 +89,24 @@ func TestSampleRing(t *testing.T) {
 	}
 }
 
+func TestSampleRingFloatST(t *testing.T) {
+	r := newSampleRing(10, 5, chunkenc.ValNone)
+	require.Empty(t, r.fBuf)
+	require.Empty(t, r.hBuf)
+	require.Empty(t, r.fhBuf)
+	require.Empty(t, r.iBuf)
+
+	r.addF(fSample{st: 100, t: 11, f: 3.14})
+	it := r.iterator()
+
+	require.Equal(t, chunkenc.ValFloat, it.Next())
+	ts, f := it.At()
+	require.Equal(t, int64(11), ts)
+	require.Equal(t, 3.14, f)
+	require.Equal(t, int64(100), it.AtST())
+	require.Equal(t, chunkenc.ValNone, it.Next())
+}
+
 func TestSampleRingMixed(t *testing.T) {
 	h1 := tsdbutil.GenerateTestHistogram(1)
 	h2 := tsdbutil.GenerateTestHistogram(2)
@@ -102,39 +119,43 @@ func TestSampleRingMixed(t *testing.T) {
 	require.Empty(t, r.iBuf)
 
 	// But then mixed adds should work as expected.
-	r.addF(fSample{t: 1, f: 3.14})
-	r.addH(hSample{t: 2, h: h1})
+	r.addF(fSample{st: 10, t: 11, f: 3.14})
+	r.addH(hSample{st: 20, t: 21, h: h1})
 
 	it := r.iterator()
 
 	require.Equal(t, chunkenc.ValFloat, it.Next())
 	ts, f := it.At()
-	require.Equal(t, int64(1), ts)
+	require.Equal(t, int64(11), ts)
 	require.Equal(t, 3.14, f)
+	require.Equal(t, int64(10), it.AtST())
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
 	var h *histogram.Histogram
 	ts, h = it.AtHistogram()
-	require.Equal(t, int64(2), ts)
+	require.Equal(t, int64(21), ts)
 	require.Equal(t, h1, h)
+	require.Equal(t, int64(20), it.AtST())
 	require.Equal(t, chunkenc.ValNone, it.Next())
 
 	r.reset()
 	it = r.iterator()
 	require.Equal(t, chunkenc.ValNone, it.Next())
 
-	r.addF(fSample{t: 3, f: 4.2})
-	r.addH(hSample{t: 4, h: h2})
+	r.addF(fSample{st: 30, t: 31, f: 4.2})
+	r.addH(hSample{st: 40, t: 41, h: h2})
 
 	it = r.iterator()
 
 	require.Equal(t, chunkenc.ValFloat, it.Next())
 	ts, f = it.At()
-	require.Equal(t, int64(3), ts)
+	require.Equal(t, int64(31), ts)
 	require.Equal(t, 4.2, f)
+	require.Equal(t, int64(30), it.AtST())
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
 	ts, h = it.AtHistogram()
-	require.Equal(t, int64(4), ts)
+	require.Equal(t, int64(41), ts)
 	require.Equal(t, h2, h)
+	require.Equal(t, int64(40), it.AtST())
 	require.Equal(t, chunkenc.ValNone, it.Next())
 }
 
@@ -160,44 +181,50 @@ func TestSampleRingAtFloatHistogram(t *testing.T) {
 	it := r.iterator()
 	require.Equal(t, chunkenc.ValNone, it.Next())
 
-	r.addFH(fhSample{t: 1, fh: fh1})
-	r.addFH(fhSample{t: 2, fh: fh2})
+	r.addFH(fhSample{st: 10, t: 11, fh: fh1})
+	r.addFH(fhSample{st: 20, t: 21, fh: fh2})
 
 	it = r.iterator()
 
 	require.Equal(t, chunkenc.ValFloatHistogram, it.Next())
 	ts, fh = it.AtFloatHistogram(fh)
-	require.Equal(t, int64(1), ts)
+	require.Equal(t, int64(11), ts)
 	require.Equal(t, fh1, fh)
+	require.Equal(t, int64(10), it.AtST())
 	require.Equal(t, chunkenc.ValFloatHistogram, it.Next())
 	ts, fh = it.AtFloatHistogram(fh)
-	require.Equal(t, int64(2), ts)
+	require.Equal(t, int64(21), ts)
 	require.Equal(t, fh2, fh)
+	require.Equal(t, int64(20), it.AtST())
 	require.Equal(t, chunkenc.ValNone, it.Next())
 
 	r.reset()
 	it = r.iterator()
 	require.Equal(t, chunkenc.ValNone, it.Next())
 
-	r.addH(hSample{t: 3, h: h1})
-	r.addH(hSample{t: 4, h: h2})
+	r.addH(hSample{st: 30, t: 31, h: h1})
+	r.addH(hSample{st: 40, t: 41, h: h2})
 
 	it = r.iterator()
 
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
 	ts, h = it.AtHistogram()
-	require.Equal(t, int64(3), ts)
+	require.Equal(t, int64(31), ts)
 	require.Equal(t, h1, h)
+	require.Equal(t, int64(30), it.AtST())
 	ts, fh = it.AtFloatHistogram(fh)
-	require.Equal(t, int64(3), ts)
+	require.Equal(t, int64(31), ts)
 	require.Equal(t, h1.ToFloat(nil), fh)
+	require.Equal(t, int64(30), it.AtST())
 	require.Equal(t, chunkenc.ValHistogram, it.Next())
 	ts, h = it.AtHistogram()
-	require.Equal(t, int64(4), ts)
+	require.Equal(t, int64(41), ts)
 	require.Equal(t, h2, h)
+	require.Equal(t, int64(40), it.AtST())
 	ts, fh = it.AtFloatHistogram(fh)
-	require.Equal(t, int64(4), ts)
+	require.Equal(t, int64(41), ts)
 	require.Equal(t, h2.ToFloat(nil), fh)
+	require.Equal(t, int64(40), it.AtST())
 	require.Equal(t, chunkenc.ValNone, it.Next())
 }
 
@@ -209,59 +236,63 @@ func TestBufferedSeriesIterator(t *testing.T) {
 		bit := it.Buffer()
 		for bit.Next() == chunkenc.ValFloat {
 			t, f := bit.At()
-			b = append(b, fSample{t: t, f: f})
+			st := bit.AtST()
+			b = append(b, fSample{st: st, t: t, f: f})
 		}
 		require.Equal(t, exp, b, "buffer mismatch")
 	}
-	sampleEq := func(ets int64, ev float64) {
+	sampleEq := func(est, ets int64, ev float64) {
 		ts, v := it.At()
+		st := it.AtST()
+		require.Equal(t, est, st, "start timestamp mismatch")
 		require.Equal(t, ets, ts, "timestamp mismatch")
 		require.Equal(t, ev, v, "value mismatch")
 	}
-	prevSampleEq := func(ets int64, ev float64, eok bool) {
+	prevSampleEq := func(est, ets int64, ev float64, eok bool) {
 		s, ok := it.PeekBack(1)
 		require.Equal(t, eok, ok, "exist mismatch")
+		require.Equal(t, est, s.ST(), "start timestamp mismatch")
 		require.Equal(t, ets, s.T(), "timestamp mismatch")
 		require.Equal(t, ev, s.F(), "value mismatch")
 	}
 
 	it = NewBufferIterator(NewListSeriesIterator(samples{
-		fSample{t: 1, f: 2},
-		fSample{t: 2, f: 3},
-		fSample{t: 3, f: 4},
-		fSample{t: 4, f: 5},
-		fSample{t: 5, f: 6},
-		fSample{t: 99, f: 8},
-		fSample{t: 100, f: 9},
-		fSample{t: 101, f: 10},
+		fSample{st: -1, t: 1, f: 2},
+		fSample{st: 1, t: 2, f: 3},
+		fSample{st: 2, t: 3, f: 4},
+		fSample{st: 3, t: 4, f: 5},
+		fSample{st: 3, t: 5, f: 6},
+		fSample{st: 50, t: 99, f: 8},
+		fSample{st: 99, t: 100, f: 9},
+		fSample{st: 100, t: 101, f: 10},
 	}), 2)
 
 	require.Equal(t, chunkenc.ValFloat, it.Seek(-123), "seek failed")
-	sampleEq(1, 2)
-	prevSampleEq(0, 0, false)
+	sampleEq(-1, 1, 2)
+	prevSampleEq(0, 0, 0, false)
 	bufferEq(nil)
 
 	require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
-	sampleEq(2, 3)
-	prevSampleEq(1, 2, true)
-	bufferEq([]fSample{{t: 1, f: 2}})
+	sampleEq(1, 2, 3)
+	prevSampleEq(-1, 1, 2, true)
+	bufferEq([]fSample{{st: -1, t: 1, f: 2}})
 
 	require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
 	require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
 	require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
-	sampleEq(5, 6)
-	prevSampleEq(4, 5, true)
-	bufferEq([]fSample{{t: 2, f: 3}, {t: 3, f: 4}, {t: 4, f: 5}})
+	sampleEq(3, 5, 6)
+	prevSampleEq(3, 4, 5, true)
+	bufferEq([]fSample{{st: 1, t: 2, f: 3}, {st: 2, t: 3, f: 4}, {st: 3, t: 4, f: 5}})
 
 	require.Equal(t, chunkenc.ValFloat, it.Seek(5), "seek failed")
-	sampleEq(5, 6)
-	prevSampleEq(4, 5, true)
-	bufferEq([]fSample{{t: 2, f: 3}, {t: 3, f: 4}, {t: 4, f: 5}})
+	sampleEq(3, 5, 6)
+	prevSampleEq(3, 4, 5, true)
+	bufferEq([]fSample{{st: 1, t: 2, f: 3}, {st: 2, t: 3, f: 4}, {st: 3, t: 4, f: 5}})
 
 	require.Equal(t, chunkenc.ValFloat, it.Seek(101), "seek failed")
-	sampleEq(101, 10)
-	prevSampleEq(100, 9, true)
-	bufferEq([]fSample{{t: 99, f: 8}, {t: 100, f: 9}})
+	sampleEq(100, 101, 10)
+	prevSampleEq(99, 100, 9, true)
+	bufferEq([]fSample{{st: 50, t: 99, f: 8}, {st: 99, t: 100, f: 9}})
 
 	require.Equal(t, chunkenc.ValNone, it.Next(), "next succeeded unexpectedly")
 	require.Equal(t, chunkenc.ValNone, it.Seek(1024), "seek succeeded unexpectedly")
@@ -402,6 +433,10 @@ func (*mockSeriesIterator) AtT() int64 {
 	return 0 // Not really mocked.
 }
 
+func (*mockSeriesIterator) AtST() int64 {
+	return 0 // Not really mocked.
+}
+
 type fakeSeriesIterator struct {
 	nsamples int64
 	step     int64
@@ -428,6 +463,10 @@ func (it *fakeSeriesIterator) AtT() int64 {
 	return it.idx * it.step
 }
 
+func (*fakeSeriesIterator) AtST() int64 {
+	return 0 // No start timestamps in this fake iterator.
+}
+
 func (it *fakeSeriesIterator) Next() chunkenc.ValueType {
 	it.idx++
 	if it.idx >= it.nsamples {

storage/fanout.go

@@ -15,6 +15,7 @@ package storage
 
 import (
 	"context"
+	"errors"
 	"log/slog"
 
 	"github.com/prometheus/common/model"
@@ -23,7 +24,6 @@ import (
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/metadata"
-	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 )
 
 type fanout struct {
@@ -82,11 +82,14 @@ func (f *fanout) Querier(mint, maxt int64) (Querier, error) {
 		querier, err := storage.Querier(mint, maxt)
 		if err != nil {
 			// Close already open Queriers, append potential errors to returned error.
-			errs := tsdb_errors.NewMulti(err, primary.Close())
-			for _, q := range secondaries {
-				errs.Add(q.Close())
+			errs := []error{
+				err,
+				primary.Close(),
 			}
-			return nil, errs.Err()
+			for _, q := range secondaries {
+				errs = append(errs, q.Close())
+			}
+			return nil, errors.Join(errs...)
 		}
 		if _, ok := querier.(noopQuerier); !ok {
 			secondaries = append(secondaries, querier)
@@ -106,11 +109,14 @@ func (f *fanout) ChunkQuerier(mint, maxt int64) (ChunkQuerier, error) {
 		querier, err := storage.ChunkQuerier(mint, maxt)
 		if err != nil {
 			// Close already open Queriers, append potential errors to returned error.
-			errs := tsdb_errors.NewMulti(err, primary.Close())
-			for _, q := range secondaries {
-				errs.Add(q.Close())
+			errs := []error{
+				err,
+				primary.Close(),
 			}
-			return nil, errs.Err()
+			for _, q := range secondaries {
+				errs = append(errs, q.Close())
+			}
+			return nil, errors.Join(errs...)
 		}
 		secondaries = append(secondaries, querier)
 	}
@@ -130,13 +136,28 @@ func (f *fanout) Appender(ctx context.Context) Appender {
 	}
 }
 
+func (f *fanout) AppenderV2(ctx context.Context) AppenderV2 {
+	primary := f.primary.AppenderV2(ctx)
+	secondaries := make([]AppenderV2, 0, len(f.secondaries))
+	for _, storage := range f.secondaries {
+		secondaries = append(secondaries, storage.AppenderV2(ctx))
+	}
+	return &fanoutAppenderV2{
+		logger:      f.logger,
+		primary:     primary,
+		secondaries: secondaries,
+	}
+}
+
 // Close closes the storage and all its underlying resources.
 func (f *fanout) Close() error {
-	errs := tsdb_errors.NewMulti(f.primary.Close())
-	for _, s := range f.secondaries {
-		errs.Add(s.Close())
+	errs := []error{
+		f.primary.Close(),
 	}
-	return errs.Err()
+	for _, s := range f.secondaries {
+		errs = append(errs, s.Close())
+	}
+	return errors.Join(errs...)
 }
 
 // fanoutAppender implements Appender.
@@ -268,5 +289,59 @@ func (f *fanoutAppender) Rollback() (err error) {
 			f.logger.Error("Squashed rollback error on rollback", "err", rollbackErr)
 		}
 	}
-	return nil
+	return err
+}
+
+type fanoutAppenderV2 struct {
+	logger *slog.Logger
+
+	primary     AppenderV2
+	secondaries []AppenderV2
+}
+
+func (f *fanoutAppenderV2) Append(ref SeriesRef, l labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts AOptions) (SeriesRef, error) {
+	ref, err := f.primary.Append(ref, l, st, t, v, h, fh, opts)
+	var partialErr AppendPartialError
+	if partialErr.Handle(err) != nil {
+		return ref, err
+	}
+
+	for _, appender := range f.secondaries {
+		if _, err := appender.Append(ref, l, st, t, v, h, fh, opts); err != nil {
+			if partialErr.Handle(err) != nil {
+				return ref, err
+			}
+		}
+	}
+	return ref, partialErr.ErrOrNil()
+}
+
+func (f *fanoutAppenderV2) Commit() (err error) {
+	err = f.primary.Commit()
+
+	for _, appender := range f.secondaries {
+		if err == nil {
+			err = appender.Commit()
+		} else {
+			if rollbackErr := appender.Rollback(); rollbackErr != nil {
+				f.logger.Error("Squashed rollback error on commit", "err", rollbackErr)
+			}
+		}
+	}
+	return err
+}
+
+func (f *fanoutAppenderV2) Rollback() (err error) {
+	err = f.primary.Rollback()
+
+	for _, appender := range f.secondaries {
+		rollbackErr := appender.Rollback()
+		switch {
+		case err == nil:
+			err = rollbackErr
+		case rollbackErr != nil:
+			f.logger.Error("Squashed rollback error on rollback", "err", rollbackErr)
+		}
+	}
+	return err
 }

storage/fanout_test.go

@@ -21,11 +21,14 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
+	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
+	"github.com/prometheus/prometheus/tsdb/tsdbutil"
 	"github.com/prometheus/prometheus/util/annotations"
 	"github.com/prometheus/prometheus/util/teststorage"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestFanout_SelectSorted(t *testing.T) {
@@ -132,6 +135,115 @@ func TestFanout_SelectSorted(t *testing.T) {
 	})
 }
 
+func TestFanout_SelectSorted_AppenderV2(t *testing.T) {
+	inputLabel := labels.FromStrings(model.MetricNameLabel, "a")
+	outputLabel := labels.FromStrings(model.MetricNameLabel, "a")
+
+	inputTotalSize := 0
+
+	priStorage := teststorage.New(t)
+	defer priStorage.Close()
+	app1 := priStorage.AppenderV2(t.Context())
+	_, err := app1.Append(0, inputLabel, 0, 0, 0, nil, nil, storage.AOptions{})
+	require.NoError(t, err)
+	inputTotalSize++
+	_, err = app1.Append(0, inputLabel, 0, 1000, 1, nil, nil, storage.AOptions{})
+	require.NoError(t, err)
+	inputTotalSize++
+	_, err = app1.Append(0, inputLabel, 0, 2000, 2, nil, nil, storage.AOptions{})
+	require.NoError(t, err)
+	inputTotalSize++
+	require.NoError(t, app1.Commit())
+
+	remoteStorage1 := teststorage.New(t)
+	defer remoteStorage1.Close()
+	app2 := remoteStorage1.AppenderV2(t.Context())
+	_, err = app2.Append(0, inputLabel, 0, 3000, 3, nil, nil, storage.AOptions{})
+	require.NoError(t, err)
+	inputTotalSize++
+	_, err = app2.Append(0, inputLabel, 0, 4000, 4, nil, nil, storage.AOptions{})
+	require.NoError(t, err)
+	inputTotalSize++
+	_, err = app2.Append(0, inputLabel, 0, 5000, 5, nil, nil, storage.AOptions{})
+	require.NoError(t, err)
+	inputTotalSize++
+	require.NoError(t, app2.Commit())
+
+	remoteStorage2 := teststorage.New(t)
+	defer remoteStorage2.Close()
+
+	app3 := remoteStorage2.AppenderV2(t.Context())
+	_, err = app3.Append(0, inputLabel, 0, 6000, 6, nil, nil, storage.AOptions{})
+	require.NoError(t, err)
+	inputTotalSize++
+	_, err = app3.Append(0, inputLabel, 0, 7000, 7, nil, nil, storage.AOptions{})
+	require.NoError(t, err)
+	inputTotalSize++
+	_, err = app3.Append(0, inputLabel, 0, 8000, 8, nil, nil, storage.AOptions{})
+	require.NoError(t, err)
+	inputTotalSize++
+
+	require.NoError(t, app3.Commit())
+
+	fanoutStorage := storage.NewFanout(nil, priStorage, remoteStorage1, remoteStorage2)
+
+	t.Run("querier", func(t *testing.T) {
+		querier, err := fanoutStorage.Querier(0, 8000)
+		require.NoError(t, err)
+		defer querier.Close()
+
+		matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "a")
+		require.NoError(t, err)
+
+		seriesSet := querier.Select(t.Context(), true, nil, matcher)
+
+		result := make(map[int64]float64)
+		var labelsResult labels.Labels
+		var iterator chunkenc.Iterator
+		for seriesSet.Next() {
+			series := seriesSet.At()
+			seriesLabels := series.Labels()
+			labelsResult = seriesLabels
+			iterator := series.Iterator(iterator)
+			for iterator.Next() == chunkenc.ValFloat {
+				timestamp, value := iterator.At()
+				result[timestamp] = value
+			}
+		}
+
+		require.Equal(t, labelsResult, outputLabel)
+		require.Len(t, result, inputTotalSize)
+	})
+	t.Run("chunk querier", func(t *testing.T) {
+		querier, err := fanoutStorage.ChunkQuerier(0, 8000)
+		require.NoError(t, err)
+		defer querier.Close()
+
+		matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "a")
+		require.NoError(t, err)
+
+		seriesSet := storage.NewSeriesSetFromChunkSeriesSet(querier.Select(t.Context(), true, nil, matcher))
+
+		result := make(map[int64]float64)
+		var labelsResult labels.Labels
+		var iterator chunkenc.Iterator
+		for seriesSet.Next() {
+			series := seriesSet.At()
+			seriesLabels := series.Labels()
+			labelsResult = seriesLabels
+			iterator := series.Iterator(iterator)
+			for iterator.Next() == chunkenc.ValFloat {
+				timestamp, value := iterator.At()
+				result[timestamp] = value
+			}
+		}
+
+		require.NoError(t, seriesSet.Err())
+		require.Equal(t, labelsResult, outputLabel)
+		require.Len(t, result, inputTotalSize)
+	})
+}
+
 func TestFanoutErrors(t *testing.T) {
 	workingStorage := teststorage.New(t)
 	defer workingStorage.Close()
@@ -224,9 +336,10 @@ type errChunkQuerier struct{ errQuerier }
 func (errStorage) ChunkQuerier(_, _ int64) (storage.ChunkQuerier, error) {
 	return errChunkQuerier{}, nil
 }
-func (errStorage) Appender(context.Context) storage.Appender { return nil }
-func (errStorage) StartTime() (int64, error)                 { return 0, nil }
-func (errStorage) Close() error                              { return nil }
+func (errStorage) Appender(context.Context) storage.Appender     { return nil }
+func (errStorage) AppenderV2(context.Context) storage.AppenderV2 { return nil }
+func (errStorage) StartTime() (int64, error)                     { return 0, nil }
+func (errStorage) Close() error                                  { return nil }
 
 func (errQuerier) Select(context.Context, bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet {
 	return storage.ErrSeriesSet(errSelect)
@@ -245,3 +358,216 @@ func (errQuerier) Close() error { return nil }
 func (errChunkQuerier) Select(context.Context, bool, *storage.SelectHints, ...*labels.Matcher) storage.ChunkSeriesSet {
 	return storage.ErrChunkSeriesSet(errSelect)
 }
+
+type mockStorage struct {
+	app   storage.Appendable
+	appV2 storage.AppendableV2
+	storage.Storage
+}
+
+func (m mockStorage) Appender(ctx context.Context) storage.Appender {
+	return m.app.Appender(ctx)
+}
+
+func (m mockStorage) AppenderV2(ctx context.Context) storage.AppenderV2 {
+	return m.appV2.AppenderV2(ctx)
+}
+
+type sample = teststorage.Sample
+
+func withoutExemplars(s []sample) (ret []sample) {
+	ret = make([]sample, len(s))
+	copy(ret, s)
+	for i := range ret {
+		ret[i].ES = nil
+	}
+	return ret
+}
+
+type fanoutAppenderTestCase struct {
+	name      string
+	primary   *teststorage.Appendable
+	secondary *teststorage.Appendable
+
+	expectAppendErr     bool
+	expectExemplarError bool
+	expectCommitError   bool
+
+	expectPrimarySamples   []sample
+	expectSecondarySamples []sample
+}
+
+func fanoutAppenderTestCases(expected []sample) []fanoutAppenderTestCase {
+	appErr := errors.New("append test error")
+	exErr := errors.New("exemplar test error")
+	commitErr := errors.New("commit test error")
+
+	return []fanoutAppenderTestCase{
+		{
+			name:      "both works",
+			primary:   teststorage.NewAppendable(),
+			secondary: teststorage.NewAppendable(),
+
+			expectPrimarySamples:   expected,
+			expectSecondarySamples: expected,
+		},
+		{
+			name:      "primary errors",
+			primary:   teststorage.NewAppendable().WithErrs(func(labels.Labels) error { return appErr }, exErr, commitErr),
+			secondary: teststorage.NewAppendable(),
+
+			expectAppendErr:     true,
+			expectExemplarError: true,
+			expectCommitError:   true,
+		},
+		{
+			name:      "exemplar errors",
+			primary:   teststorage.NewAppendable().WithErrs(func(labels.Labels) error { return nil }, exErr, nil),
+			secondary: teststorage.NewAppendable().WithErrs(func(labels.Labels) error { return nil }, exErr, nil),
+
+			expectAppendErr:     false,
+			expectExemplarError: true,
+			expectCommitError:   false,
+
+			expectPrimarySamples:   withoutExemplars(expected),
+			expectSecondarySamples: withoutExemplars(expected),
+		},
+		{
+			name:      "secondary errors",
+			primary:   teststorage.NewAppendable(),
+			secondary: teststorage.NewAppendable().WithErrs(func(labels.Labels) error { return appErr }, exErr, commitErr),
+
+			expectAppendErr:     true,
+			expectExemplarError: true,
+			expectCommitError:   true,
+
+			expectPrimarySamples: expected,
+		},
+	}
+}
+
+func TestFanoutAppender(t *testing.T) {
+	h := tsdbutil.GenerateTestHistogram(0)
+	fh := tsdbutil.GenerateTestFloatHistogram(0)
+	ex := exemplar.Exemplar{Value: 1}
+
+	expected := []sample{
+		{L: labels.FromStrings(model.MetricNameLabel, "metric1"), V: 1, ES: []exemplar.Exemplar{ex}},
+		{L: labels.FromStrings(model.MetricNameLabel, "metric2"), T: 1, H: h},
+		{L: labels.FromStrings(model.MetricNameLabel, "metric3"), T: 2, FH: fh},
+	}
+	for _, tt := range fanoutAppenderTestCases(expected) {
+		t.Run(tt.name, func(t *testing.T) {
+			f := storage.NewFanout(nil, mockStorage{app: tt.primary}, mockStorage{app: tt.secondary})
+
+			app := f.Appender(t.Context())
+			ref, err := app.Append(0, labels.FromStrings(model.MetricNameLabel, "metric1"), 0, 1)
+			if tt.expectAppendErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			_, err = app.AppendExemplar(ref, labels.FromStrings(model.MetricNameLabel, "metric1"), ex)
+			if tt.expectExemplarError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			_, err = app.AppendHistogram(0, labels.FromStrings(model.MetricNameLabel, "metric2"), 1, h, nil)
+			if tt.expectAppendErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			_, err = app.AppendHistogram(0, labels.FromStrings(model.MetricNameLabel, "metric3"), 2, nil, fh)
+			if tt.expectAppendErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			err = app.Commit()
+			if tt.expectCommitError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			require.Nil(t, tt.primary.PendingSamples())
+			testutil.RequireEqual(t, tt.expectPrimarySamples, tt.primary.ResultSamples())
+			require.Nil(t, tt.primary.RolledbackSamples())
+
+			require.Nil(t, tt.secondary.PendingSamples())
+			testutil.RequireEqual(t, tt.expectSecondarySamples, tt.secondary.ResultSamples())
+			require.Nil(t, tt.secondary.RolledbackSamples())
+		})
+	}
+}
+
+func TestFanoutAppenderV2(t *testing.T) {
+	h := tsdbutil.GenerateTestHistogram(0)
+	fh := tsdbutil.GenerateTestFloatHistogram(0)
+	ex := exemplar.Exemplar{Value: 1}
+
+	expected := []sample{
+		{L: labels.FromStrings(model.MetricNameLabel, "metric1"), ST: -1, V: 1, ES: []exemplar.Exemplar{ex}},
+		{L: labels.FromStrings(model.MetricNameLabel, "metric2"), ST: -2, T: 1, H: h},
+		{L: labels.FromStrings(model.MetricNameLabel, "metric3"), ST: -3, T: 2, FH: fh},
+	}
+
+	for _, tt := range fanoutAppenderTestCases(expected) {
+		t.Run(tt.name, func(t *testing.T) {
+			f := storage.NewFanout(nil, mockStorage{appV2: tt.primary}, mockStorage{appV2: tt.secondary})
+
+			app := f.AppenderV2(t.Context())
+			_, err := app.Append(0, labels.FromStrings(model.MetricNameLabel, "metric1"), -1, 0, 1, nil, nil, storage.AOptions{
+				Exemplars: []exemplar.Exemplar{ex},
+			})
+			switch {
+			case tt.expectAppendErr:
+				require.Error(t, err)
+			case tt.expectExemplarError:
+				var pErr *storage.AppendPartialError
+				require.ErrorAs(t, err, &pErr)
+				// One for primary, one for secondary.
+				// This is because in V2 flow we must append sample even when first append partially failed with exemplars.
+				// Filtering out exemplars is neither feasible, nor important.
+				require.Len(t, pErr.ExemplarErrors, 2)
+			default:
+				require.NoError(t, err)
+			}
+
+			_, err = app.Append(0, labels.FromStrings(model.MetricNameLabel, "metric2"), -2, 1, 0, h, nil, storage.AOptions{})
+			if tt.expectAppendErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			_, err = app.Append(0, labels.FromStrings(model.MetricNameLabel, "metric3"), -3, 2, 0, nil, fh, storage.AOptions{})
+			if tt.expectAppendErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			err = app.Commit()
+			if tt.expectCommitError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			require.Nil(t, tt.primary.PendingSamples())
+			testutil.RequireEqual(t, tt.expectPrimarySamples, tt.primary.ResultSamples())
+			require.Nil(t, tt.primary.RolledbackSamples())
+
+			require.Nil(t, tt.secondary.PendingSamples())
+			testutil.RequireEqual(t, tt.expectSecondarySamples, tt.secondary.ResultSamples())
+			require.Nil(t, tt.secondary.RolledbackSamples())
+		})
+	}
+}

storage/interface.go

@@ -61,7 +61,8 @@ type SeriesRef uint64
 
 // Appendable allows creating Appender.
 //
-// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026).
+// WARNING(bwplotka): Switch to AppendableV2 is in progress (https://github.com/prometheus/prometheus/issues/17632).
+// Appendable will be removed soon (ETA: Q2 2026).
 type Appendable interface {
 	// Appender returns a new appender for the storage.
 	//
@@ -77,10 +78,16 @@ type SampleAndChunkQueryable interface {
 }
 
 // Storage ingests and manages samples, along with various indexes. All methods
-// are goroutine-safe. Storage implements storage.Appender.
+// are goroutine-safe.
 type Storage interface {
 	SampleAndChunkQueryable
+
+	// Appendable allows appending to storage.
+	// WARNING(bwplotka): Switch to AppendableV2 is in progress (https://github.com/prometheus/prometheus/issues/17632).
+	// Appendable will be removed soon (ETA: Q2 2026).
 	Appendable
+	// AppendableV2 allows appending to storage.
+	AppendableV2
 
 	// StartTime returns the oldest timestamp stored in the storage.
 	StartTime() (int64, error)
@@ -261,7 +268,8 @@ func (f QueryableFunc) Querier(mint, maxt int64) (Querier, error) {
 
 // AppendOptions provides options for implementations of the Appender interface.
 //
-// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026).
+// WARNING(bwplotka): Switch to AppendableV2 is in progress (https://github.com/prometheus/prometheus/issues/17632).
+// AppendOptions will be removed soon (ETA: Q2 2026).
 type AppendOptions struct {
 	// DiscardOutOfOrder tells implementation that this append should not be out
 	// of order. An OOO append MUST be rejected with storage.ErrOutOfOrderSample
@@ -278,7 +286,8 @@ type AppendOptions struct {
 // I.e. timestamp order within batch is not validated, samples are not reordered per timestamp or by float/histogram
 // type.
 //
-// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026).
+// WARNING(bwplotka): Switch to AppendableV2 is in progress (https://github.com/prometheus/prometheus/issues/17632).
+// Appender will be removed soon (ETA: Q2 2026).
 type Appender interface {
 	AppenderTransaction
 
@@ -315,7 +324,8 @@ type GetRef interface {
 // ExemplarAppender provides an interface for adding samples to exemplar storage, which
 // within Prometheus is in-memory only.
 //
-// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026).
+// WARNING(bwplotka): Switch to AppendableV2 is in progress (https://github.com/prometheus/prometheus/issues/17632).
+// ExemplarAppender will be removed soon (ETA: Q2 2026).
 type ExemplarAppender interface {
 	// AppendExemplar adds an exemplar for the given series labels.
 	// An optional reference number can be provided to accelerate calls.
@@ -333,7 +343,8 @@ type ExemplarAppender interface {
 
 // HistogramAppender provides an interface for appending histograms to the storage.
 //
-// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026).
+// WARNING(bwplotka): Switch to AppendableV2 is in progress (https://github.com/prometheus/prometheus/issues/17632).
+// HistogramAppender will be removed soon (ETA: Q2 2026).
 type HistogramAppender interface {
 	// AppendHistogram adds a histogram for the given series labels. An
 	// optional reference number can be provided to accelerate calls. A
@@ -365,7 +376,8 @@ type HistogramAppender interface {
 
 // MetadataUpdater provides an interface for associating metadata to stored series.
 //
-// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026).
+// WARNING(bwplotka): Switch to AppendableV2 is in progress (https://github.com/prometheus/prometheus/issues/17632).
+// MetadataUpdater will be removed soon (ETA: Q2 2026).
 type MetadataUpdater interface {
 	// UpdateMetadata updates a metadata entry for the given series and labels.
 	// A series reference number is returned which can be used to modify the
@@ -379,7 +391,8 @@ type MetadataUpdater interface {
 
 // StartTimestampAppender provides an interface for appending ST to storage.
 //
-// WARNING: Work AppendableV2 is in progress. Appendable will be removed soon (ETA: Q2 2026).
+// WARNING(bwplotka): Switch to AppendableV2 is in progress (https://github.com/prometheus/prometheus/issues/17632).
+// StartTimestampAppender will be removed soon (ETA: Q2 2026).
 type StartTimestampAppender interface {
 	// AppendSTZeroSample adds synthetic zero sample for the given st timestamp,
 	// which will be associated with given series, labels and the incoming
@@ -473,9 +486,10 @@ type Series interface {
 }
 
 type mockSeries struct {
-	timestamps []int64
-	values     []float64
-	labelSet   []string
+	startTimestamps []int64
+	timestamps      []int64
+	values          []float64
+	labelSet        []string
 }
 
 func (s mockSeries) Labels() labels.Labels {
@@ -483,15 +497,19 @@ func (s mockSeries) Labels() labels.Labels {
 }
 
 func (s mockSeries) Iterator(chunkenc.Iterator) chunkenc.Iterator {
-	return chunkenc.MockSeriesIterator(s.timestamps, s.values)
+	return chunkenc.MockSeriesIterator(s.startTimestamps, s.timestamps, s.values)
 }
 
-// MockSeries returns a series with custom timestamps, values and labelSet.
-func MockSeries(timestamps []int64, values []float64, labelSet []string) Series {
+// MockSeries returns a series with custom start timestamp, timestamps, values,
+// and labelSet.
+// Start timestamps is optional, pass nil or empty slice to indicate no start
+// timestamps.
+func MockSeries(startTimestamps, timestamps []int64, values []float64, labelSet []string) Series {
 	return mockSeries{
-		timestamps: timestamps,
-		values:     values,
-		labelSet:   labelSet,
+		startTimestamps: startTimestamps,
+		timestamps:      timestamps,
+		values:          values,
+		labelSet:        labelSet,
 	}
 }
 

storage/interface_append.go

@@ -69,6 +69,7 @@ type AppendV2Options struct {
 	// Exemplars (optional) attached to the appended sample.
 	// Exemplar slice MUST be sorted by Exemplar.TS.
 	// Exemplar slice is unsafe for reuse.
+	// Duplicate exemplars errors MUST be ignored by implementations.
 	Exemplars []exemplar.Exemplar
 
 	// RejectOutOfOrder tells implementation that this append should not be out
@@ -96,6 +97,31 @@ func (e *AppendPartialError) Error() string {
 	return errs.Error()
 }
 
+// ErrOrNil returns AppendPartialError as error, returning nil
+// if there are no errors.
+func (e *AppendPartialError) ErrOrNil() error {
+	if len(e.ExemplarErrors) == 0 {
+		return nil
+	}
+	return e
+}
+
+// Handle handles the given err that may be an AppendPartialError.
+// If the err is nil or not an AppendPartialError it returns err.
+// Otherwise, partial errors are aggregated.
+func (e *AppendPartialError) Handle(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	var pErr *AppendPartialError
+	if !errors.As(err, &pErr) {
+		return err
+	}
+	e.ExemplarErrors = append(e.ExemplarErrors, pErr.ExemplarErrors...)
+	return nil
+}
+
 var _ error = &AppendPartialError{}
 
 // AppenderV2 provides appends against a storage for all types of samples.

storage/interface_test.go

@@ -23,7 +23,7 @@ import (
 )
 
 func TestMockSeries(t *testing.T) {
-	s := storage.MockSeries([]int64{1, 2, 3}, []float64{1, 2, 3}, []string{"__name__", "foo"})
+	s := storage.MockSeries(nil, []int64{1, 2, 3}, []float64{1, 2, 3}, []string{"__name__", "foo"})
 	it := s.Iterator(nil)
 	ts := []int64{}
 	vs := []float64{}
@@ -35,3 +35,20 @@ func TestMockSeries(t *testing.T) {
 	require.Equal(t, []int64{1, 2, 3}, ts)
 	require.Equal(t, []float64{1, 2, 3}, vs)
 }
+
+func TestMockSeriesWithST(t *testing.T) {
+	s := storage.MockSeries([]int64{0, 1, 2}, []int64{1, 2, 3}, []float64{1, 2, 3}, []string{"__name__", "foo"})
+	it := s.Iterator(nil)
+	ts := []int64{}
+	vs := []float64{}
+	st := []int64{}
+	for it.Next() == chunkenc.ValFloat {
+		t, v := it.At()
+		ts = append(ts, t)
+		vs = append(vs, v)
+		st = append(st, it.AtST())
+	}
+	require.Equal(t, []int64{1, 2, 3}, ts)
+	require.Equal(t, []float64{1, 2, 3}, vs)
+	require.Equal(t, []int64{0, 1, 2}, st)
+}

storage/merge.go

@@ -17,6 +17,7 @@ import (
 	"bytes"
 	"container/heap"
 	"context"
+	"errors"
 	"fmt"
 	"math"
 	"sync"
@@ -25,7 +26,6 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/chunks"
-	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 	"github.com/prometheus/prometheus/util/annotations"
 )
 
@@ -269,13 +269,13 @@ func (q *mergeGenericQuerier) LabelNames(ctx context.Context, hints *LabelHints,
 
 // Close releases the resources of the generic querier.
 func (q *mergeGenericQuerier) Close() error {
-	errs := tsdb_errors.NewMulti()
+	var errs []error
 	for _, querier := range q.queriers {
 		if err := querier.Close(); err != nil {
-			errs.Add(err)
+			errs = append(errs, err)
 		}
 	}
-	return errs.Err()
+	return errors.Join(errs...)
 }
 
 func truncateToLimit(s []string, hints *LabelHints) []string {
@@ -599,6 +599,13 @@ func (c *chainSampleIterator) AtT() int64 {
 	return c.curr.AtT()
 }
 
+func (c *chainSampleIterator) AtST() int64 {
+	if c.curr == nil {
+		panic("chainSampleIterator.AtST called before first .Next or after .Next returned false.")
+	}
+	return c.curr.AtST()
+}
+
 func (c *chainSampleIterator) Next() chunkenc.ValueType {
 	var (
 		currT           int64
@@ -679,11 +686,11 @@ func (c *chainSampleIterator) Next() chunkenc.ValueType {
 }
 
 func (c *chainSampleIterator) Err() error {
-	errs := tsdb_errors.NewMulti()
+	var errs []error
 	for _, iter := range c.iterators {
-		errs.Add(iter.Err())
+		errs = append(errs, iter.Err())
 	}
-	return errs.Err()
+	return errors.Join(errs...)
 }
 
 type samplesIteratorHeap []chunkenc.Iterator
@@ -821,12 +828,12 @@ func (c *compactChunkIterator) Next() bool {
 }
 
 func (c *compactChunkIterator) Err() error {
-	errs := tsdb_errors.NewMulti()
+	var errs []error
 	for _, iter := range c.iterators {
-		errs.Add(iter.Err())
+		errs = append(errs, iter.Err())
 	}
-	errs.Add(c.err)
-	return errs.Err()
+	errs = append(errs, c.err)
+	return errors.Join(errs...)
 }
 
 type chunkIteratorHeap []chunks.Iterator
@@ -904,9 +911,9 @@ func (c *concatenatingChunkIterator) Next() bool {
 }
 
 func (c *concatenatingChunkIterator) Err() error {
-	errs := tsdb_errors.NewMulti()
+	var errs []error
 	for _, iter := range c.iterators {
-		errs.Add(iter.Err())
+		errs = append(errs, iter.Err())
 	}
-	return errs.Err()
+	return errors.Join(errs...)
 }

storage/merge_test.go

@@ -66,116 +66,116 @@ func TestMergeQuerierWithChainMerger(t *testing.T) {
 		{
 			name: "one querier, two series",
 			querierSeries: [][]Series{{
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}}),
 			}},
 			expected: NewMockSeriesSet(
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}}),
 			),
 		},
 		{
 			name: "two queriers, one different series each",
 			querierSeries: [][]Series{{
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}}),
 			}, {
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}}),
 			}},
 			expected: NewMockSeriesSet(
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}}),
 			),
 		},
 		{
 			name: "two time unsorted queriers, two series each",
 			querierSeries: [][]Series{{
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}, fSample{6, 6}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 5, 5}, fSample{0, 6, 6}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}}),
 			}, {
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}, fSample{4, 4}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 3, 3}, fSample{0, 4, 4}}),
 			}},
 			expected: NewMockSeriesSet(
 				NewListSeries(
 					labels.FromStrings("bar", "baz"),
-					[]chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}, fSample{6, 6}},
+					[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}, fSample{0, 6, 6}},
 				),
 				NewListSeries(
 					labels.FromStrings("foo", "bar"),
-					[]chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}},
+					[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 4, 4}},
 				),
 			),
 		},
 		{
 			name: "five queriers, only two queriers have two time unsorted series each",
 			querierSeries: [][]Series{{}, {}, {
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}, fSample{6, 6}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 5, 5}, fSample{0, 6, 6}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}}),
 			}, {
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}, fSample{4, 4}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 3, 3}, fSample{0, 4, 4}}),
 			}, {}},
 			expected: NewMockSeriesSet(
 				NewListSeries(
 					labels.FromStrings("bar", "baz"),
-					[]chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}, fSample{6, 6}},
+					[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}, fSample{0, 6, 6}},
 				),
 				NewListSeries(
 					labels.FromStrings("foo", "bar"),
-					[]chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}},
+					[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 4, 4}},
 				),
 			),
 		},
 		{
 			name: "two queriers, only two queriers have two time unsorted series each, with 3 noop and one nil querier together",
 			querierSeries: [][]Series{{}, {}, {
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}, fSample{6, 6}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 5, 5}, fSample{0, 6, 6}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}}),
 			}, {
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}, fSample{4, 4}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 3, 3}, fSample{0, 4, 4}}),
 			}, {}},
 			extraQueriers: []Querier{NoopQuerier(), NoopQuerier(), nil, NoopQuerier()},
 			expected: NewMockSeriesSet(
 				NewListSeries(
 					labels.FromStrings("bar", "baz"),
-					[]chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}, fSample{6, 6}},
+					[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}, fSample{0, 6, 6}},
 				),
 				NewListSeries(
 					labels.FromStrings("foo", "bar"),
-					[]chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}},
+					[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 4, 4}},
 				),
 			),
 		},
 		{
 			name: "two queriers, with two series, one is overlapping",
 			querierSeries: [][]Series{{}, {}, {
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 21}, fSample{3, 31}, fSample{5, 5}, fSample{6, 6}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 2, 21}, fSample{0, 3, 31}, fSample{0, 5, 5}, fSample{0, 6, 6}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}}),
 			}, {
-				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 22}, fSample{3, 32}}),
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}, fSample{4, 4}}),
+				NewListSeries(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 22}, fSample{0, 3, 32}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 3, 3}, fSample{0, 4, 4}}),
 			}, {}},
 			expected: NewMockSeriesSet(
 				NewListSeries(
 					labels.FromStrings("bar", "baz"),
-					[]chunks.Sample{fSample{1, 1}, fSample{2, 21}, fSample{3, 31}, fSample{5, 5}, fSample{6, 6}},
+					[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 21}, fSample{0, 3, 31}, fSample{0, 5, 5}, fSample{0, 6, 6}},
 				),
 				NewListSeries(
 					labels.FromStrings("foo", "bar"),
-					[]chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}},
+					[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 4, 4}},
 				),
 			),
 		},
 		{
 			name: "two queries, one with NaN samples series",
 			querierSeries: [][]Series{{
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, math.NaN()}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, math.NaN()}}),
 			}, {
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{1, 1}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 1, 1}}),
 			}},
 			expected: NewMockSeriesSet(
-				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, math.NaN()}, fSample{1, 1}}),
+				NewListSeries(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, math.NaN()}, fSample{0, 1, 1}}),
 			),
 		},
 	} {
@@ -249,108 +249,108 @@ func TestMergeChunkQuerierWithNoVerticalChunkSeriesMerger(t *testing.T) {
 		{
 			name: "one querier, two series",
 			chkQuerierSeries: [][]ChunkSeries{{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}}, []chunks.Sample{fSample{0, 2, 2}}),
 			}},
 			expected: NewMockChunkSeriesSet(
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}}, []chunks.Sample{fSample{0, 2, 2}}),
 			),
 		},
 		{
 			name: "two secondaries, one different series each",
 			chkQuerierSeries: [][]ChunkSeries{{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
 			}, {
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}}, []chunks.Sample{fSample{0, 2, 2}}),
 			}},
 			expected: NewMockChunkSeriesSet(
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}}, []chunks.Sample{fSample{0, 2, 2}}),
 			),
 		},
 		{
 			name: "two secondaries, two not in time order series each",
 			chkQuerierSeries: [][]ChunkSeries{{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}}, []chunks.Sample{fSample{6, 6}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 5, 5}}, []chunks.Sample{fSample{0, 6, 6}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}}, []chunks.Sample{fSample{0, 2, 2}}),
 			}, {
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}}, []chunks.Sample{fSample{4, 4}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 3, 3}}, []chunks.Sample{fSample{0, 4, 4}}),
 			}},
 			expected: NewMockChunkSeriesSet(
 				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
-					[]chunks.Sample{fSample{1, 1}, fSample{2, 2}},
-					[]chunks.Sample{fSample{3, 3}},
-					[]chunks.Sample{fSample{5, 5}},
-					[]chunks.Sample{fSample{6, 6}},
+					[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}},
+					[]chunks.Sample{fSample{0, 3, 3}},
+					[]chunks.Sample{fSample{0, 5, 5}},
+					[]chunks.Sample{fSample{0, 6, 6}},
 				),
 				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"),
-					[]chunks.Sample{fSample{0, 0}, fSample{1, 1}},
-					[]chunks.Sample{fSample{2, 2}},
-					[]chunks.Sample{fSample{3, 3}},
-					[]chunks.Sample{fSample{4, 4}},
+					[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}},
+					[]chunks.Sample{fSample{0, 2, 2}},
+					[]chunks.Sample{fSample{0, 3, 3}},
+					[]chunks.Sample{fSample{0, 4, 4}},
 				),
 			),
 		},
 		{
 			name: "five secondaries, only two have two not in time order series each",
 			chkQuerierSeries: [][]ChunkSeries{{}, {}, {
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}}, []chunks.Sample{fSample{6, 6}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 5, 5}}, []chunks.Sample{fSample{0, 6, 6}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}}, []chunks.Sample{fSample{0, 2, 2}}),
 			}, {
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}}, []chunks.Sample{fSample{4, 4}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 3, 3}}, []chunks.Sample{fSample{0, 4, 4}}),
 			}, {}},
 			expected: NewMockChunkSeriesSet(
 				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
-					[]chunks.Sample{fSample{1, 1}, fSample{2, 2}},
-					[]chunks.Sample{fSample{3, 3}},
-					[]chunks.Sample{fSample{5, 5}},
-					[]chunks.Sample{fSample{6, 6}},
+					[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}},
+					[]chunks.Sample{fSample{0, 3, 3}},
+					[]chunks.Sample{fSample{0, 5, 5}},
+					[]chunks.Sample{fSample{0, 6, 6}},
 				),
 				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"),
-					[]chunks.Sample{fSample{0, 0}, fSample{1, 1}},
-					[]chunks.Sample{fSample{2, 2}},
-					[]chunks.Sample{fSample{3, 3}},
-					[]chunks.Sample{fSample{4, 4}},
+					[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}},
+					[]chunks.Sample{fSample{0, 2, 2}},
+					[]chunks.Sample{fSample{0, 3, 3}},
+					[]chunks.Sample{fSample{0, 4, 4}},
 				),
 			),
 		},
 		{
 			name: "two secondaries, with two not in time order series each, with 3 noop queries and one nil together",
 			chkQuerierSeries: [][]ChunkSeries{{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{5, 5}}, []chunks.Sample{fSample{6, 6}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}}, []chunks.Sample{fSample{2, 2}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 5, 5}}, []chunks.Sample{fSample{0, 6, 6}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}}, []chunks.Sample{fSample{0, 2, 2}}),
 			}, {
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{3, 3}}, []chunks.Sample{fSample{4, 4}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 3, 3}}, []chunks.Sample{fSample{0, 4, 4}}),
 			}},
 			extraQueriers: []ChunkQuerier{NoopChunkedQuerier(), NoopChunkedQuerier(), nil, NoopChunkedQuerier()},
 			expected: NewMockChunkSeriesSet(
 				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
-					[]chunks.Sample{fSample{1, 1}, fSample{2, 2}},
-					[]chunks.Sample{fSample{3, 3}},
-					[]chunks.Sample{fSample{5, 5}},
-					[]chunks.Sample{fSample{6, 6}},
+					[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}},
+					[]chunks.Sample{fSample{0, 3, 3}},
+					[]chunks.Sample{fSample{0, 5, 5}},
+					[]chunks.Sample{fSample{0, 6, 6}},
 				),
 				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"),
-					[]chunks.Sample{fSample{0, 0}, fSample{1, 1}},
-					[]chunks.Sample{fSample{2, 2}},
-					[]chunks.Sample{fSample{3, 3}},
-					[]chunks.Sample{fSample{4, 4}},
+					[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}},
+					[]chunks.Sample{fSample{0, 2, 2}},
+					[]chunks.Sample{fSample{0, 3, 3}},
+					[]chunks.Sample{fSample{0, 4, 4}},
 				),
 			),
 		},
 		{
 			name: "two queries, one with NaN samples series",
 			chkQuerierSeries: [][]ChunkSeries{{
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, math.NaN()}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, math.NaN()}}),
 			}, {
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{1, 1}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 1, 1}}),
 			}},
 			expected: NewMockChunkSeriesSet(
-				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, math.NaN()}}, []chunks.Sample{fSample{1, 1}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("foo", "bar"), []chunks.Sample{fSample{0, 0, math.NaN()}}, []chunks.Sample{fSample{0, 1, 1}}),
 			),
 		},
 	} {
@@ -387,13 +387,13 @@ func TestMergeChunkQuerierWithNoVerticalChunkSeriesMerger(t *testing.T) {
 func histogramSample(ts int64, hint histogram.CounterResetHint) hSample {
 	h := tsdbutil.GenerateTestHistogram(ts + 1)
 	h.CounterResetHint = hint
-	return hSample{t: ts, h: h}
+	return hSample{st: -ts, t: ts, h: h}
 }
 
 func floatHistogramSample(ts int64, hint histogram.CounterResetHint) fhSample {
 	fh := tsdbutil.GenerateTestFloatHistogram(ts + 1)
 	fh.CounterResetHint = hint
-	return fhSample{t: ts, fh: fh}
+	return fhSample{st: -ts, t: ts, fh: fh}
 }
 
 // Shorthands for counter reset hints.
@@ -431,9 +431,9 @@ func TestCompactingChunkSeriesMerger(t *testing.T) {
 		{
 			name: "single series",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
 			},
-			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
+			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
 		},
 		{
 			name: "two empty series",
@@ -446,55 +446,55 @@ func TestCompactingChunkSeriesMerger(t *testing.T) {
 		{
 			name: "two non overlapping",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{5, 5}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}, fSample{0, 5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 7, 7}, fSample{0, 9, 9}}, []chunks.Sample{fSample{0, 10, 10}}),
 			},
-			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{5, 5}}, []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}),
+			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}, fSample{0, 5, 5}}, []chunks.Sample{fSample{0, 7, 7}, fSample{0, 9, 9}}, []chunks.Sample{fSample{0, 10, 10}}),
 		},
 		{
 			name: "two overlapping",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{8, 8}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}, fSample{0, 8, 8}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 7, 7}, fSample{0, 9, 9}}, []chunks.Sample{fSample{0, 10, 10}}),
 			},
-			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{7, 7}, fSample{8, 8}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}),
+			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}, fSample{0, 7, 7}, fSample{0, 8, 8}, fSample{0, 9, 9}}, []chunks.Sample{fSample{0, 10, 10}}),
 		},
 		{
 			name: "two duplicated",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}}),
 			},
-			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}),
+			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}}),
 		},
 		{
 			name: "three overlapping",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{6, 6}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{4, 4}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 6, 6}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 4, 4}}),
 			},
-			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}, fSample{5, 5}, fSample{6, 6}}),
+			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 4, 4}, fSample{0, 5, 5}, fSample{0, 6, 6}}),
 		},
 		{
 			name: "three in chained overlap",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{4, 4}, fSample{6, 66}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{6, 6}, fSample{10, 10}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 4, 4}, fSample{0, 6, 66}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 6, 6}, fSample{0, 10, 10}}),
 			},
-			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{4, 4}, fSample{5, 5}, fSample{6, 66}, fSample{10, 10}}),
+			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 4, 4}, fSample{0, 5, 5}, fSample{0, 6, 66}, fSample{0, 10, 10}}),
 		},
 		{
 			name: "three in chained overlap complex",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{5, 5}}, []chunks.Sample{fSample{10, 10}, fSample{15, 15}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{20, 20}}, []chunks.Sample{fSample{25, 25}, fSample{30, 30}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{18, 18}, fSample{26, 26}}, []chunks.Sample{fSample{31, 31}, fSample{35, 35}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 5, 5}}, []chunks.Sample{fSample{0, 10, 10}, fSample{0, 15, 15}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 2, 2}, fSample{0, 20, 20}}, []chunks.Sample{fSample{0, 25, 25}, fSample{0, 30, 30}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 18, 18}, fSample{0, 26, 26}}, []chunks.Sample{fSample{0, 31, 31}, fSample{0, 35, 35}}),
 			},
 			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
-				[]chunks.Sample{fSample{0, 0}, fSample{2, 2}, fSample{5, 5}, fSample{10, 10}, fSample{15, 15}, fSample{18, 18}, fSample{20, 20}, fSample{25, 25}, fSample{26, 26}, fSample{30, 30}},
-				[]chunks.Sample{fSample{31, 31}, fSample{35, 35}},
+				[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 2, 2}, fSample{0, 5, 5}, fSample{0, 10, 10}, fSample{0, 15, 15}, fSample{0, 18, 18}, fSample{0, 20, 20}, fSample{0, 25, 25}, fSample{0, 26, 26}, fSample{0, 30, 30}},
+				[]chunks.Sample{fSample{0, 31, 31}, fSample{0, 35, 35}},
 			),
 		},
 		{
@@ -534,13 +534,13 @@ func TestCompactingChunkSeriesMerger(t *testing.T) {
 			name: "histogram chunks overlapping with float chunks",
 			input: []ChunkSeries{
 				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{histogramSample(0), histogramSample(5)}, []chunks.Sample{histogramSample(10), histogramSample(15)}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{12, 12}}, []chunks.Sample{fSample{14, 14}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 12, 12}}, []chunks.Sample{fSample{0, 14, 14}}),
 			},
 			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
 				[]chunks.Sample{histogramSample(0)},
-				[]chunks.Sample{fSample{1, 1}},
+				[]chunks.Sample{fSample{0, 1, 1}},
 				[]chunks.Sample{histogramSample(5), histogramSample(10)},
-				[]chunks.Sample{fSample{12, 12}, fSample{14, 14}},
+				[]chunks.Sample{fSample{0, 12, 12}, fSample{0, 14, 14}},
 				[]chunks.Sample{histogramSample(15)},
 			),
 		},
@@ -560,13 +560,13 @@ func TestCompactingChunkSeriesMerger(t *testing.T) {
 			name: "float histogram chunks overlapping with float chunks",
 			input: []ChunkSeries{
 				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{floatHistogramSample(0), floatHistogramSample(5)}, []chunks.Sample{floatHistogramSample(10), floatHistogramSample(15)}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{12, 12}}, []chunks.Sample{fSample{14, 14}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 12, 12}}, []chunks.Sample{fSample{0, 14, 14}}),
 			},
 			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
 				[]chunks.Sample{floatHistogramSample(0)},
-				[]chunks.Sample{fSample{1, 1}},
+				[]chunks.Sample{fSample{0, 1, 1}},
 				[]chunks.Sample{floatHistogramSample(5), floatHistogramSample(10)},
-				[]chunks.Sample{fSample{12, 12}, fSample{14, 14}},
+				[]chunks.Sample{fSample{0, 12, 12}, fSample{0, 14, 14}},
 				[]chunks.Sample{floatHistogramSample(15)},
 			),
 		},
@@ -736,9 +736,9 @@ func TestConcatenatingChunkSeriesMerger(t *testing.T) {
 		{
 			name: "single series",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
 			},
-			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}}),
+			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}}),
 		},
 		{
 			name: "two empty series",
@@ -751,70 +751,70 @@ func TestConcatenatingChunkSeriesMerger(t *testing.T) {
 		{
 			name: "two non overlapping",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{5, 5}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}, fSample{0, 5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 7, 7}, fSample{0, 9, 9}}, []chunks.Sample{fSample{0, 10, 10}}),
 			},
-			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{5, 5}}, []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}),
+			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}, fSample{0, 5, 5}}, []chunks.Sample{fSample{0, 7, 7}, fSample{0, 9, 9}}, []chunks.Sample{fSample{0, 10, 10}}),
 		},
 		{
 			name: "two overlapping",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{8, 8}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}, fSample{0, 8, 8}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 7, 7}, fSample{0, 9, 9}}, []chunks.Sample{fSample{0, 10, 10}}),
 			},
 			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
-				[]chunks.Sample{fSample{1, 1}, fSample{2, 2}}, []chunks.Sample{fSample{3, 3}, fSample{8, 8}},
-				[]chunks.Sample{fSample{7, 7}, fSample{9, 9}}, []chunks.Sample{fSample{10, 10}},
+				[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}}, []chunks.Sample{fSample{0, 3, 3}, fSample{0, 8, 8}},
+				[]chunks.Sample{fSample{0, 7, 7}, fSample{0, 9, 9}}, []chunks.Sample{fSample{0, 10, 10}},
 			),
 		},
 		{
 			name: "two duplicated",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}}),
 			},
 			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
-				[]chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}},
-				[]chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{5, 5}},
+				[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}},
+				[]chunks.Sample{fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}},
 			),
 		},
 		{
 			name: "three overlapping",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{6, 6}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{4, 4}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 6, 6}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 4, 4}}),
 			},
 			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
-				[]chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}},
-				[]chunks.Sample{fSample{2, 2}, fSample{3, 3}, fSample{6, 6}},
-				[]chunks.Sample{fSample{0, 0}, fSample{4, 4}},
+				[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}},
+				[]chunks.Sample{fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 6, 6}},
+				[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 4, 4}},
 			),
 		},
 		{
 			name: "three in chained overlap",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{4, 4}, fSample{6, 66}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{6, 6}, fSample{10, 10}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 4, 4}, fSample{0, 6, 66}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 6, 6}, fSample{0, 10, 10}}),
 			},
 			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
-				[]chunks.Sample{fSample{1, 1}, fSample{2, 2}, fSample{3, 3}, fSample{5, 5}},
-				[]chunks.Sample{fSample{4, 4}, fSample{6, 66}},
-				[]chunks.Sample{fSample{6, 6}, fSample{10, 10}},
+				[]chunks.Sample{fSample{0, 1, 1}, fSample{0, 2, 2}, fSample{0, 3, 3}, fSample{0, 5, 5}},
+				[]chunks.Sample{fSample{0, 4, 4}, fSample{0, 6, 66}},
+				[]chunks.Sample{fSample{0, 6, 6}, fSample{0, 10, 10}},
 			),
 		},
 		{
 			name: "three in chained overlap complex",
 			input: []ChunkSeries{
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0}, fSample{5, 5}}, []chunks.Sample{fSample{10, 10}, fSample{15, 15}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{2, 2}, fSample{20, 20}}, []chunks.Sample{fSample{25, 25}, fSample{30, 30}}),
-				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{18, 18}, fSample{26, 26}}, []chunks.Sample{fSample{31, 31}, fSample{35, 35}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 0, 0}, fSample{0, 5, 5}}, []chunks.Sample{fSample{0, 10, 10}, fSample{0, 15, 15}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 2, 2}, fSample{0, 20, 20}}, []chunks.Sample{fSample{0, 25, 25}, fSample{0, 30, 30}}),
+				NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"), []chunks.Sample{fSample{0, 18, 18}, fSample{0, 26, 26}}, []chunks.Sample{fSample{0, 31, 31}, fSample{0, 35, 35}}),
 			},
 			expected: NewListChunkSeriesFromSamples(labels.FromStrings("bar", "baz"),
-				[]chunks.Sample{fSample{0, 0}, fSample{5, 5}}, []chunks.Sample{fSample{10, 10}, fSample{15, 15}},
-				[]chunks.Sample{fSample{2, 2}, fSample{20, 20}}, []chunks.Sample{fSample{25, 25}, fSample{30, 30}},
-				[]chunks.Sample{fSample{18, 18}, fSample{26, 26}}, []chunks.Sample{fSample{31, 31}, fSample{35, 35}},
+				[]chunks.Sample{fSample{0, 0, 0}, fSample{0, 5, 5}}, []chunks.Sample{fSample{0, 10, 10}, fSample{0, 15, 15}},
+				[]chunks.Sample{fSample{0, 2, 2}, fSample{0, 20, 20}}, []chunks.Sample{fSample{0, 25, 25}, fSample{0, 30, 30}},
+				[]chunks.Sample{fSample{0, 18, 18}, fSample{0, 26, 26}}, []chunks.Sample{fSample{0, 31, 31}, fSample{0, 35, 35}},
 			),
 		},
 		{
@@ -1059,7 +1059,7 @@ func (*mockChunkSeriesSet) Warnings() annotations.Annotations { return nil }
 
 func TestChainSampleIterator(t *testing.T) {
 	for sampleType, sampleFunc := range map[string]func(int64) chunks.Sample{
-		"float":           func(ts int64) chunks.Sample { return fSample{ts, float64(ts)} },
+		"float":           func(ts int64) chunks.Sample { return fSample{-ts, ts, float64(ts)} },
 		"histogram":       func(ts int64) chunks.Sample { return histogramSample(ts, uk) },
 		"float histogram": func(ts int64) chunks.Sample { return floatHistogramSample(ts, uk) },
 	} {
@@ -1176,7 +1176,7 @@ func TestChainSampleIteratorHistogramCounterResetHint(t *testing.T) {
 
 func TestChainSampleIteratorSeek(t *testing.T) {
 	for sampleType, sampleFunc := range map[string]func(int64) chunks.Sample{
-		"float":           func(ts int64) chunks.Sample { return fSample{ts, float64(ts)} },
+		"float":           func(ts int64) chunks.Sample { return fSample{-ts, ts, float64(ts)} },
 		"histogram":       func(ts int64) chunks.Sample { return histogramSample(ts, uk) },
 		"float histogram": func(ts int64) chunks.Sample { return floatHistogramSample(ts, uk) },
 	} {
@@ -1224,13 +1224,13 @@ func TestChainSampleIteratorSeek(t *testing.T) {
 				switch merged.Seek(tc.seek) {
 				case chunkenc.ValFloat:
 					t, f := merged.At()
-					actual = append(actual, fSample{t, f})
+					actual = append(actual, fSample{merged.AtST(), t, f})
 				case chunkenc.ValHistogram:
 					t, h := merged.AtHistogram(nil)
-					actual = append(actual, hSample{t, h})
+					actual = append(actual, hSample{merged.AtST(), t, h})
 				case chunkenc.ValFloatHistogram:
 					t, fh := merged.AtFloatHistogram(nil)
-					actual = append(actual, fhSample{t, fh})
+					actual = append(actual, fhSample{merged.AtST(), t, fh})
 				}
 				s, err := ExpandSamples(merged, nil)
 				require.NoError(t, err)
@@ -1243,7 +1243,7 @@ func TestChainSampleIteratorSeek(t *testing.T) {
 
 func TestChainSampleIteratorSeekFailingIterator(t *testing.T) {
 	merged := ChainSampleIteratorFromIterators(nil, []chunkenc.Iterator{
-		NewListSeriesIterator(samples{fSample{0, 0.1}, fSample{1, 1.1}, fSample{2, 2.1}}),
+		NewListSeriesIterator(samples{fSample{0, 0, 0.1}, fSample{0, 1, 1.1}, fSample{0, 2, 2.1}}),
 		errIterator{errors.New("something went wrong")},
 	})
 
@@ -1253,7 +1253,7 @@ func TestChainSampleIteratorSeekFailingIterator(t *testing.T) {
 
 func TestChainSampleIteratorNextImmediatelyFailingIterator(t *testing.T) {
 	merged := ChainSampleIteratorFromIterators(nil, []chunkenc.Iterator{
-		NewListSeriesIterator(samples{fSample{0, 0.1}, fSample{1, 1.1}, fSample{2, 2.1}}),
+		NewListSeriesIterator(samples{fSample{0, 0, 0.1}, fSample{0, 1, 1.1}, fSample{0, 2, 2.1}}),
 		errIterator{errors.New("something went wrong")},
 	})
 
@@ -1263,7 +1263,7 @@ func TestChainSampleIteratorNextImmediatelyFailingIterator(t *testing.T) {
 	// Next() does some special handling for the first iterator, so make sure it handles the first iterator returning an error too.
 	merged = ChainSampleIteratorFromIterators(nil, []chunkenc.Iterator{
 		errIterator{errors.New("something went wrong")},
-		NewListSeriesIterator(samples{fSample{0, 0.1}, fSample{1, 1.1}, fSample{2, 2.1}}),
+		NewListSeriesIterator(samples{fSample{0, 0, 0.1}, fSample{0, 1, 1.1}, fSample{0, 2, 2.1}}),
 	})
 
 	require.Equal(t, chunkenc.ValNone, merged.Next())
@@ -1310,13 +1310,13 @@ func TestChainSampleIteratorSeekHistogramCounterResetHint(t *testing.T) {
 				switch merged.Seek(tc.seek) {
 				case chunkenc.ValFloat:
 					t, f := merged.At()
-					actual = append(actual, fSample{t, f})
+					actual = append(actual, fSample{merged.AtST(), t, f})
 				case chunkenc.ValHistogram:
 					t, h := merged.AtHistogram(nil)
-					actual = append(actual, hSample{t, h})
+					actual = append(actual, hSample{merged.AtST(), t, h})
 				case chunkenc.ValFloatHistogram:
 					t, fh := merged.AtFloatHistogram(nil)
-					actual = append(actual, fhSample{t, fh})
+					actual = append(actual, fhSample{merged.AtST(), t, fh})
 				}
 				s, err := ExpandSamples(merged, nil)
 				require.NoError(t, err)
@@ -1716,6 +1716,10 @@ func (errIterator) AtT() int64 {
 	return 0
 }
 
+func (errIterator) AtST() int64 {
+	return 0
+}
+
 func (e errIterator) Err() error {
 	return e.err
 }

storage/remote/codec.go

@@ -564,6 +564,12 @@ func (c *concreteSeriesIterator) AtT() int64 {
 	return c.series.floats[c.floatsCur].Timestamp
 }
 
+// TODO(krajorama): implement AtST. Maybe. concreteSeriesIterator is used
+// for turning query results into an iterable, but query results do not have ST.
+func (*concreteSeriesIterator) AtST() int64 {
+	return 0
+}
+
 const noTS = int64(math.MaxInt64)
 
 // Next implements chunkenc.Iterator.
@@ -832,6 +838,11 @@ func (it *chunkedSeriesIterator) AtT() int64 {
 	return it.cur.AtT()
 }
 
+// TODO(krajorama): test AtST once we have a chunk format that provides ST.
+func (it *chunkedSeriesIterator) AtST() int64 {
+	return it.cur.AtST()
+}
+
 func (it *chunkedSeriesIterator) Err() error {
 	return it.err
 }

storage/remote/codec_test.go

@@ -1146,7 +1146,7 @@ func buildTestChunks(t *testing.T) []prompb.Chunk {
 		minTimeMs := time
 
 		for j := range numSamplesPerTestChunk {
-			a.Append(time, float64(i+j))
+			a.Append(0, time, float64(i+j))
 			time += int64(1000)
 		}
 

storage/remote/otlptranslator/prometheusremotewrite/helper.go

@@ -19,6 +19,7 @@ package prometheusremotewrite
 import (
 	"context"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"log"
 	"math"
@@ -32,7 +33,7 @@ import (
 	"github.com/prometheus/otlptranslator"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
-	conventions "go.opentelemetry.io/collector/semconv/v1.6.1"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
 
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/labels"
@@ -63,15 +64,14 @@ const (
 // createAttributes creates a slice of Prometheus Labels with OTLP attributes and pairs of string values.
 // Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen and
 // if logOnOverwrite is true, the overwrite is logged. Resulting label names are sanitized.
-// If settings.PromoteResourceAttributes is not empty, it's a set of resource attributes that should be promoted to labels.
-func (c *PrometheusConverter) createAttributes(resource pcommon.Resource, attributes pcommon.Map, scope scope, settings Settings,
+//
+// This function requires for cached resource and scope labels to be set up first.
+func (c *PrometheusConverter) createAttributes(attributes pcommon.Map, settings Settings,
 	ignoreAttrs []string, logOnOverwrite bool, meta Metadata, extras ...string,
 ) (labels.Labels, error) {
-	resourceAttrs := resource.Attributes()
-	serviceName, haveServiceName := resourceAttrs.Get(conventions.AttributeServiceName)
-	instance, haveInstanceID := resourceAttrs.Get(conventions.AttributeServiceInstanceID)
-
-	promoteScope := settings.PromoteScopeMetadata && scope.name != ""
+	if c.resourceLabels == nil {
+		return labels.EmptyLabels(), errors.New("createAttributes called without initializing resource context")
+	}
 
 	// Ensure attributes are sorted by key for consistent merging of keys which
 	// collide when sanitized.
@@ -88,12 +88,6 @@ func (c *PrometheusConverter) createAttributes(resource pcommon.Resource, attrib
 	c.scratchBuilder.Sort()
 	sortedLabels := c.scratchBuilder.Labels()
 
-	labelNamer := otlptranslator.LabelNamer{
-		UTF8Allowed:                 settings.AllowUTF8,
-		UnderscoreLabelSanitization: settings.LabelNameUnderscoreSanitization,
-		PreserveMultipleUnderscores: settings.LabelNamePreserveMultipleUnderscores,
-	}
-
 	if settings.AllowUTF8 {
 		// UTF8 is allowed, so conflicts aren't possible.
 		c.builder.Reset(sortedLabels)
@@ -106,7 +100,7 @@ func (c *PrometheusConverter) createAttributes(resource pcommon.Resource, attrib
 			if sortErr != nil {
 				return
 			}
-			finalKey, err := labelNamer.Build(l.Name)
+			finalKey, err := c.buildLabelName(l.Name)
 			if err != nil {
 				sortErr = err
 				return
@@ -122,28 +116,36 @@ func (c *PrometheusConverter) createAttributes(resource pcommon.Resource, attrib
 		}
 	}
 
-	err := settings.PromoteResourceAttributes.addPromotedAttributes(c.builder, resourceAttrs, labelNamer)
-	if err != nil {
-		return labels.EmptyLabels(), err
-	}
-	if promoteScope {
-		var rangeErr error
-		scope.attributes.Range(func(k string, v pcommon.Value) bool {
-			name, err := labelNamer.Build("otel_scope_" + k)
-			if err != nil {
-				rangeErr = err
-				return false
+	if settings.PromoteResourceAttributes != nil {
+		// Merge cached promoted resource labels.
+		c.resourceLabels.promotedLabels.Range(func(l labels.Label) {
+			if c.builder.Get(l.Name) == "" {
+				c.builder.Set(l.Name, l.Value)
 			}
-			c.builder.Set(name, v.AsString())
-			return true
 		})
-		if rangeErr != nil {
-			return labels.EmptyLabels(), rangeErr
+	}
+	// Merge cached job/instance labels.
+	if c.resourceLabels.jobLabel != "" {
+		c.builder.Set(model.JobLabel, c.resourceLabels.jobLabel)
+	}
+	if c.resourceLabels.instanceLabel != "" {
+		c.builder.Set(model.InstanceLabel, c.resourceLabels.instanceLabel)
+	}
+	// Merge cached external labels.
+	for key, value := range c.resourceLabels.externalLabels {
+		if c.builder.Get(key) == "" {
+			c.builder.Set(key, value)
 		}
-		// Scope Name, Version and Schema URL are added after attributes to ensure they are not overwritten by attributes.
-		c.builder.Set("otel_scope_name", scope.name)
-		c.builder.Set("otel_scope_version", scope.version)
-		c.builder.Set("otel_scope_schema_url", scope.schemaURL)
+	}
+
+	if c.scopeLabels != nil {
+		// Merge cached scope labels if scope promotion is enabled.
+		c.scopeLabels.scopeAttrs.Range(func(l labels.Label) {
+			c.builder.Set(l.Name, l.Value)
+		})
+		c.builder.Set("otel_scope_name", c.scopeLabels.scopeName)
+		c.builder.Set("otel_scope_version", c.scopeLabels.scopeVersion)
+		c.builder.Set("otel_scope_schema_url", c.scopeLabels.scopeSchemaURL)
 	}
 
 	if settings.EnableTypeAndUnitLabels {
@@ -156,27 +158,6 @@ func (c *PrometheusConverter) createAttributes(resource pcommon.Resource, attrib
 		}
 	}
 
-	// Map service.name + service.namespace to job.
-	if haveServiceName {
-		val := serviceName.AsString()
-		if serviceNamespace, ok := resourceAttrs.Get(conventions.AttributeServiceNamespace); ok {
-			val = fmt.Sprintf("%s/%s", serviceNamespace.AsString(), val)
-		}
-		c.builder.Set(model.JobLabel, val)
-	}
-	// Map service.instance.id to instance.
-	if haveInstanceID {
-		c.builder.Set(model.InstanceLabel, instance.AsString())
-	}
-	for key, value := range settings.ExternalLabels {
-		// External labels have already been sanitized.
-		if existingValue := c.builder.Get(key); existingValue != "" {
-			// Skip external labels if they are overridden by metric attributes.
-			continue
-		}
-		c.builder.Set(key, value)
-	}
-
 	for i := 0; i < len(extras); i += 2 {
 		if i+1 >= len(extras) {
 			break
@@ -189,7 +170,7 @@ func (c *PrometheusConverter) createAttributes(resource pcommon.Resource, attrib
 		// internal labels should be maintained.
 		if len(name) <= 4 || name[:2] != "__" || name[len(name)-2:] != "__" {
 			var err error
-			name, err = labelNamer.Build(name)
+			name, err = c.buildLabelName(name)
 			if err != nil {
 				return labels.EmptyLabels(), err
 			}
@@ -223,7 +204,7 @@ func aggregationTemporality(metric pmetric.Metric) (pmetric.AggregationTemporali
 // However, work is under way to resolve this shortcoming through a feature called native histograms custom buckets:
 // https://github.com/prometheus/prometheus/issues/13485.
 func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPoints pmetric.HistogramDataPointSlice,
-	resource pcommon.Resource, settings Settings, scope scope, meta Metadata,
+	settings Settings, meta Metadata,
 ) error {
 	for x := 0; x < dataPoints.Len(); x++ {
 		if err := c.everyN.checkContext(ctx); err != nil {
@@ -233,7 +214,7 @@ func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPo
 		pt := dataPoints.At(x)
 		timestamp := convertTimeStamp(pt.Timestamp())
 		startTimestamp := convertTimeStamp(pt.StartTimestamp())
-		baseLabels, err := c.createAttributes(resource, pt.Attributes(), scope, settings, nil, false, meta)
+		baseLabels, err := c.createAttributes(pt.Attributes(), settings, nil, false, meta)
 		if err != nil {
 			return err
 		}
@@ -424,8 +405,8 @@ func findMinAndMaxTimestamps(metric pmetric.Metric, minTimestamp, maxTimestamp p
 	return minTimestamp, maxTimestamp
 }
 
-func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoints pmetric.SummaryDataPointSlice, resource pcommon.Resource,
-	settings Settings, scope scope, meta Metadata,
+func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoints pmetric.SummaryDataPointSlice,
+	settings Settings, meta Metadata,
 ) error {
 	for x := 0; x < dataPoints.Len(); x++ {
 		if err := c.everyN.checkContext(ctx); err != nil {
@@ -435,7 +416,7 @@ func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoin
 		pt := dataPoints.At(x)
 		timestamp := convertTimeStamp(pt.Timestamp())
 		startTimestamp := convertTimeStamp(pt.StartTimestamp())
-		baseLabels, err := c.createAttributes(resource, pt.Attributes(), scope, settings, nil, false, meta)
+		baseLabels, err := c.createAttributes(pt.Attributes(), settings, nil, false, meta)
 		if err != nil {
 			return err
 		}
@@ -504,9 +485,9 @@ func (c *PrometheusConverter) addResourceTargetInfo(resource pcommon.Resource, s
 
 	attributes := resource.Attributes()
 	identifyingAttrs := []string{
-		conventions.AttributeServiceNamespace,
-		conventions.AttributeServiceName,
-		conventions.AttributeServiceInstanceID,
+		string(semconv.ServiceNamespaceKey),
+		string(semconv.ServiceNameKey),
+		string(semconv.ServiceInstanceIDKey),
 	}
 	nonIdentifyingAttrsCount := attributes.Len()
 	for _, a := range identifyingAttrs {
@@ -538,7 +519,12 @@ func (c *PrometheusConverter) addResourceTargetInfo(resource pcommon.Resource, s
 		MetricFamilyName: name,
 	}
 	// TODO: should target info have the __type__ metadata label?
-	lbls, err := c.createAttributes(resource, attributes, scope{}, settings, identifyingAttrs, false, Metadata{}, model.MetricNameLabel, name)
+	// target_info is a resource-level metric and should not include scope labels.
+	// Temporarily clear scope labels for this call.
+	savedScopeLabels := c.scopeLabels
+	c.scopeLabels = nil
+	lbls, err := c.createAttributes(attributes, settings, identifyingAttrs, false, Metadata{}, model.MetricNameLabel, name)
+	c.scopeLabels = savedScopeLabels
 	if err != nil {
 		return err
 	}

storage/remote/otlptranslator/prometheusremotewrite/helper_test.go

@@ -413,7 +413,11 @@ func TestCreateAttributes(t *testing.T) {
 			if tc.attrs != (pcommon.Map{}) {
 				testAttrs = tc.attrs
 			}
-			lbls, err := c.createAttributes(testResource, testAttrs, tc.scope, settings, tc.ignoreAttrs, false, Metadata{}, model.MetricNameLabel, "test_metric")
+			// Initialize resource and scope context as FromMetrics would.
+			require.NoError(t, c.setResourceContext(testResource, settings))
+			require.NoError(t, c.setScopeContext(tc.scope, settings))
+
+			lbls, err := c.createAttributes(testAttrs, settings, tc.ignoreAttrs, false, Metadata{}, model.MetricNameLabel, "test_metric")
 			require.NoError(t, err)
 
 			testutil.RequireEqual(t, tc.expectedLabels, lbls)
@@ -643,15 +647,19 @@ func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) {
 			metric := tt.metric()
 			mockAppender := &mockCombinedAppender{}
 			converter := NewPrometheusConverter(mockAppender)
+			settings := Settings{
+				PromoteScopeMetadata: tt.promoteScope,
+			}
+			resource := pcommon.NewResource()
+
+			// Initialize resource and scope context as FromMetrics would.
+			require.NoError(t, converter.setResourceContext(resource, settings))
+			require.NoError(t, converter.setScopeContext(tt.scope, settings))
 
 			converter.addSummaryDataPoints(
 				context.Background(),
 				metric.Summary().DataPoints(),
-				pcommon.NewResource(),
-				Settings{
-					PromoteScopeMetadata: tt.promoteScope,
-				},
-				tt.scope,
+				settings,
 				Metadata{
 					MetricFamilyName: metric.Name(),
 				},
@@ -806,15 +814,19 @@ func TestPrometheusConverter_AddHistogramDataPoints(t *testing.T) {
 			metric := tt.metric()
 			mockAppender := &mockCombinedAppender{}
 			converter := NewPrometheusConverter(mockAppender)
+			settings := Settings{
+				PromoteScopeMetadata: tt.promoteScope,
+			}
+			resource := pcommon.NewResource()
+
+			// Initialize resource and scope context as FromMetrics would.
+			require.NoError(t, converter.setResourceContext(resource, settings))
+			require.NoError(t, converter.setScopeContext(tt.scope, settings))
 
 			converter.addHistogramDataPoints(
 				context.Background(),
 				metric.Histogram().DataPoints(),
-				pcommon.NewResource(),
-				Settings{
-					PromoteScopeMetadata: tt.promoteScope,
-				},
-				tt.scope,
+				settings,
 				Metadata{
 					MetricFamilyName: metric.Name(),
 				},

storage/remote/otlptranslator/prometheusremotewrite/histograms.go

@@ -22,7 +22,6 @@ import (
 	"math"
 
 	"github.com/prometheus/common/model"
-	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 
 	"github.com/prometheus/prometheus/model/histogram"
@@ -35,8 +34,7 @@ const defaultZeroThreshold = 1e-128
 // addExponentialHistogramDataPoints adds OTel exponential histogram data points to the corresponding time series
 // as native histogram samples.
 func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Context, dataPoints pmetric.ExponentialHistogramDataPointSlice,
-	resource pcommon.Resource, settings Settings, temporality pmetric.AggregationTemporality,
-	scope scope, meta Metadata,
+	settings Settings, temporality pmetric.AggregationTemporality, meta Metadata,
 ) (annotations.Annotations, error) {
 	var annots annotations.Annotations
 	for x := 0; x < dataPoints.Len(); x++ {
@@ -53,9 +51,7 @@ func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Cont
 		}
 
 		lbls, err := c.createAttributes(
-			resource,
 			pt.Attributes(),
-			scope,
 			settings,
 			nil,
 			true,
@@ -253,8 +249,7 @@ func convertBucketsLayout(bucketCounts []uint64, offset, scaleDown int32, adjust
 }
 
 func (c *PrometheusConverter) addCustomBucketsHistogramDataPoints(ctx context.Context, dataPoints pmetric.HistogramDataPointSlice,
-	resource pcommon.Resource, settings Settings, temporality pmetric.AggregationTemporality,
-	scope scope, meta Metadata,
+	settings Settings, temporality pmetric.AggregationTemporality, meta Metadata,
 ) (annotations.Annotations, error) {
 	var annots annotations.Annotations
 
@@ -272,9 +267,7 @@ func (c *PrometheusConverter) addCustomBucketsHistogramDataPoints(ctx context.Co
 		}
 
 		lbls, err := c.createAttributes(
-			resource,
 			pt.Attributes(),
-			scope,
 			settings,
 			nil,
 			true,

storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go

@@ -861,15 +861,20 @@ func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) {
 			}
 			name, err := namer.Build(TranslatorMetricFromOtelMetric(metric))
 			require.NoError(t, err)
+			settings := Settings{
+				PromoteScopeMetadata: tt.promoteScope,
+			}
+			resource := pcommon.NewResource()
+
+			// Initialize resource and scope context as FromMetrics would.
+			require.NoError(t, converter.setResourceContext(resource, settings))
+			require.NoError(t, converter.setScopeContext(tt.scope, settings))
+
 			annots, err := converter.addExponentialHistogramDataPoints(
 				context.Background(),
 				metric.ExponentialHistogram().DataPoints(),
-				pcommon.NewResource(),
-				Settings{
-					PromoteScopeMetadata: tt.promoteScope,
-				},
+				settings,
 				pmetric.AggregationTemporalityCumulative,
-				tt.scope,
 				Metadata{
 					MetricFamilyName: name,
 				},
@@ -1334,16 +1339,21 @@ func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) {
 			}
 			name, err := namer.Build(TranslatorMetricFromOtelMetric(metric))
 			require.NoError(t, err)
+			settings := Settings{
+				ConvertHistogramsToNHCB: true,
+				PromoteScopeMetadata:    tt.promoteScope,
+			}
+			resource := pcommon.NewResource()
+
+			// Initialize resource and scope context as FromMetrics would.
+			require.NoError(t, converter.setResourceContext(resource, settings))
+			require.NoError(t, converter.setScopeContext(tt.scope, settings))
+
 			annots, err := converter.addCustomBucketsHistogramDataPoints(
 				context.Background(),
 				metric.Histogram().DataPoints(),
-				pcommon.NewResource(),
-				Settings{
-					ConvertHistogramsToNHCB: true,
-					PromoteScopeMetadata:    tt.promoteScope,
-				},
+				settings,
 				pmetric.AggregationTemporalityCumulative,
-				tt.scope,
 				Metadata{
 					MetricFamilyName: name,
 				},

storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go

@@ -26,7 +26,7 @@ import (
 	"github.com/prometheus/otlptranslator"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
-	"go.uber.org/multierr"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
 
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/labels"
@@ -62,6 +62,24 @@ type Settings struct {
 	LabelNamePreserveMultipleUnderscores bool
 }
 
+// cachedResourceLabels holds precomputed labels constant for all datapoints in a ResourceMetrics.
+// These are computed once per ResourceMetrics boundary and reused for all datapoints.
+type cachedResourceLabels struct {
+	jobLabel       string        // from service.name + service.namespace.
+	instanceLabel  string        // from service.instance.id.
+	promotedLabels labels.Labels // promoted resource attributes.
+	externalLabels map[string]string
+}
+
+// cachedScopeLabels holds precomputed scope metadata labels.
+// These are computed once per ScopeMetrics boundary and reused for all datapoints.
+type cachedScopeLabels struct {
+	scopeName      string
+	scopeVersion   string
+	scopeSchemaURL string
+	scopeAttrs     labels.Labels // otel_scope_* labels.
+}
+
 // PrometheusConverter converts from OTel write format to Prometheus remote write format.
 type PrometheusConverter struct {
 	everyN         everyNTimes
@@ -70,6 +88,15 @@ type PrometheusConverter struct {
 	appender       CombinedAppender
 	// seenTargetInfo tracks target_info samples within a batch to prevent duplicates.
 	seenTargetInfo map[targetInfoKey]struct{}
+
+	// Label caching for optimization - computed once per resource/scope boundary.
+	resourceLabels *cachedResourceLabels
+	scopeLabels    *cachedScopeLabels
+	labelNamer     otlptranslator.LabelNamer
+
+	// sanitizedLabels caches the results of label name sanitization within a request.
+	// This avoids repeated string allocations for the same label names.
+	sanitizedLabels map[string]string
 }
 
 // targetInfoKey uniquely identifies a target_info sample by its labelset and timestamp.
@@ -80,12 +107,27 @@ type targetInfoKey struct {
 
 func NewPrometheusConverter(appender CombinedAppender) *PrometheusConverter {
 	return &PrometheusConverter{
-		scratchBuilder: labels.NewScratchBuilder(0),
-		builder:        labels.NewBuilder(labels.EmptyLabels()),
-		appender:       appender,
+		scratchBuilder:  labels.NewScratchBuilder(0),
+		builder:         labels.NewBuilder(labels.EmptyLabels()),
+		appender:        appender,
+		sanitizedLabels: make(map[string]string, 64), // Pre-size for typical label count.
 	}
 }
 
+// buildLabelName returns a sanitized label name, using the cache to avoid repeated allocations.
+func (c *PrometheusConverter) buildLabelName(label string) (string, error) {
+	if sanitized, ok := c.sanitizedLabels[label]; ok {
+		return sanitized, nil
+	}
+
+	sanitized, err := c.labelNamer.Build(label)
+	if err != nil {
+		return "", err
+	}
+	c.sanitizedLabels[label] = sanitized
+	return sanitized, nil
+}
+
 func TranslatorMetricFromOtelMetric(metric pmetric.Metric) otlptranslator.Metric {
 	m := otlptranslator.Metric{
 		Name: metric.Name(),
@@ -140,23 +182,33 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 	c.seenTargetInfo = make(map[targetInfoKey]struct{})
 	resourceMetricsSlice := md.ResourceMetrics()
 
-	for i := 0; i < resourceMetricsSlice.Len(); i++ {
+	for i := range resourceMetricsSlice.Len() {
 		resourceMetrics := resourceMetricsSlice.At(i)
 		resource := resourceMetrics.Resource()
 		scopeMetricsSlice := resourceMetrics.ScopeMetrics()
+		if err := c.setResourceContext(resource, settings); err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+
 		// keep track of the earliest and latest timestamp in the ResourceMetrics for
 		// use with the "target" info metric
 		earliestTimestamp := pcommon.Timestamp(math.MaxUint64)
 		latestTimestamp := pcommon.Timestamp(0)
-		for j := 0; j < scopeMetricsSlice.Len(); j++ {
+		for j := range scopeMetricsSlice.Len() {
 			scopeMetrics := scopeMetricsSlice.At(j)
 			scope := newScopeFromScopeMetrics(scopeMetrics)
+			if err := c.setScopeContext(scope, settings); err != nil {
+				errs = errors.Join(errs, err)
+				continue
+			}
+
 			metricSlice := scopeMetrics.Metrics()
 
 			// TODO: decide if instrumentation library information should be exported as labels
 			for k := 0; k < metricSlice.Len(); k++ {
 				if err := c.everyN.checkContext(ctx); err != nil {
-					errs = multierr.Append(errs, err)
+					errs = errors.Join(errs, err)
 					return annots, errs
 				}
 
@@ -164,7 +216,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 				earliestTimestamp, latestTimestamp = findMinAndMaxTimestamps(metric, earliestTimestamp, latestTimestamp)
 				temporality, hasTemporality, err := aggregationTemporality(metric)
 				if err != nil {
-					errs = multierr.Append(errs, err)
+					errs = errors.Join(errs, err)
 					continue
 				}
 
@@ -175,13 +227,13 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 					//nolint:staticcheck // QF1001 Applying De Morgan’s law would make the conditions harder to read.
 					!(temporality == pmetric.AggregationTemporalityCumulative ||
 						(settings.AllowDeltaTemporality && temporality == pmetric.AggregationTemporalityDelta)) {
-					errs = multierr.Append(errs, fmt.Errorf("invalid temporality and type combination for metric %q", metric.Name()))
+					errs = errors.Join(errs, fmt.Errorf("invalid temporality and type combination for metric %q", metric.Name()))
 					continue
 				}
 
 				promName, err := namer.Build(TranslatorMetricFromOtelMetric(metric))
 				if err != nil {
-					errs = multierr.Append(errs, err)
+					errs = errors.Join(errs, err)
 					continue
 				}
 				meta := Metadata{
@@ -199,11 +251,11 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 				case pmetric.MetricTypeGauge:
 					dataPoints := metric.Gauge().DataPoints()
 					if dataPoints.Len() == 0 {
-						errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
+						errs = errors.Join(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
 						break
 					}
-					if err := c.addGaugeNumberDataPoints(ctx, dataPoints, resource, settings, scope, meta); err != nil {
-						errs = multierr.Append(errs, err)
+					if err := c.addGaugeNumberDataPoints(ctx, dataPoints, settings, meta); err != nil {
+						errs = errors.Join(errs, err)
 						if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 							return annots, errs
 						}
@@ -211,11 +263,11 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 				case pmetric.MetricTypeSum:
 					dataPoints := metric.Sum().DataPoints()
 					if dataPoints.Len() == 0 {
-						errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
+						errs = errors.Join(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
 						break
 					}
-					if err := c.addSumNumberDataPoints(ctx, dataPoints, resource, settings, scope, meta); err != nil {
-						errs = multierr.Append(errs, err)
+					if err := c.addSumNumberDataPoints(ctx, dataPoints, settings, meta); err != nil {
+						errs = errors.Join(errs, err)
 						if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 							return annots, errs
 						}
@@ -223,23 +275,23 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 				case pmetric.MetricTypeHistogram:
 					dataPoints := metric.Histogram().DataPoints()
 					if dataPoints.Len() == 0 {
-						errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
+						errs = errors.Join(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
 						break
 					}
 					if settings.ConvertHistogramsToNHCB {
 						ws, err := c.addCustomBucketsHistogramDataPoints(
-							ctx, dataPoints, resource, settings, temporality, scope, meta,
+							ctx, dataPoints, settings, temporality, meta,
 						)
 						annots.Merge(ws)
 						if err != nil {
-							errs = multierr.Append(errs, err)
+							errs = errors.Join(errs, err)
 							if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 								return annots, errs
 							}
 						}
 					} else {
-						if err := c.addHistogramDataPoints(ctx, dataPoints, resource, settings, scope, meta); err != nil {
-							errs = multierr.Append(errs, err)
+						if err := c.addHistogramDataPoints(ctx, dataPoints, settings, meta); err != nil {
+							errs = errors.Join(errs, err)
 							if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 								return annots, errs
 							}
@@ -248,21 +300,19 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 				case pmetric.MetricTypeExponentialHistogram:
 					dataPoints := metric.ExponentialHistogram().DataPoints()
 					if dataPoints.Len() == 0 {
-						errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
+						errs = errors.Join(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
 						break
 					}
 					ws, err := c.addExponentialHistogramDataPoints(
 						ctx,
 						dataPoints,
-						resource,
 						settings,
 						temporality,
-						scope,
 						meta,
 					)
 					annots.Merge(ws)
 					if err != nil {
-						errs = multierr.Append(errs, err)
+						errs = errors.Join(errs, err)
 						if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 							return annots, errs
 						}
@@ -270,17 +320,17 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 				case pmetric.MetricTypeSummary:
 					dataPoints := metric.Summary().DataPoints()
 					if dataPoints.Len() == 0 {
-						errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
+						errs = errors.Join(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
 						break
 					}
-					if err := c.addSummaryDataPoints(ctx, dataPoints, resource, settings, scope, meta); err != nil {
-						errs = multierr.Append(errs, err)
+					if err := c.addSummaryDataPoints(ctx, dataPoints, settings, meta); err != nil {
+						errs = errors.Join(errs, err)
 						if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 							return annots, errs
 						}
 					}
 				default:
-					errs = multierr.Append(errs, errors.New("unsupported metric type"))
+					errs = errors.Join(errs, errors.New("unsupported metric type"))
 				}
 			}
 		}
@@ -288,7 +338,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 			// We have at least one metric sample for this resource.
 			// Generate a corresponding target_info series.
 			if err := c.addResourceTargetInfo(resource, settings, earliestTimestamp.AsTime(), latestTimestamp.AsTime()); err != nil {
-				errs = multierr.Append(errs, err)
+				errs = errors.Join(errs, err)
 			}
 		}
 	}
@@ -311,8 +361,11 @@ func NewPromoteResourceAttributes(otlpCfg config.OTLPConfig) *PromoteResourceAtt
 	}
 }
 
+// LabelNameBuilder is a function that builds/sanitizes label names.
+type LabelNameBuilder func(string) (string, error)
+
 // addPromotedAttributes adds labels for promoted resourceAttributes to the builder.
-func (s *PromoteResourceAttributes) addPromotedAttributes(builder *labels.Builder, resourceAttributes pcommon.Map, labelNamer otlptranslator.LabelNamer) error {
+func (s *PromoteResourceAttributes) addPromotedAttributes(builder *labels.Builder, resourceAttributes pcommon.Map, buildLabelName LabelNameBuilder) error {
 	if s == nil {
 		return nil
 	}
@@ -322,13 +375,11 @@ func (s *PromoteResourceAttributes) addPromotedAttributes(builder *labels.Builde
 		resourceAttributes.Range(func(name string, value pcommon.Value) bool {
 			if _, exists := s.attrs[name]; !exists {
 				var normalized string
-				normalized, err = labelNamer.Build(name)
+				normalized, err = buildLabelName(name)
 				if err != nil {
 					return false
 				}
-				if builder.Get(normalized) == "" {
-					builder.Set(normalized, value.AsString())
-				}
+				builder.Set(normalized, value.AsString())
 			}
 			return true
 		})
@@ -338,15 +389,91 @@ func (s *PromoteResourceAttributes) addPromotedAttributes(builder *labels.Builde
 	resourceAttributes.Range(func(name string, value pcommon.Value) bool {
 		if _, exists := s.attrs[name]; exists {
 			var normalized string
-			normalized, err = labelNamer.Build(name)
+			normalized, err = buildLabelName(name)
 			if err != nil {
 				return false
 			}
-			if builder.Get(normalized) == "" {
-				builder.Set(normalized, value.AsString())
-			}
+			builder.Set(normalized, value.AsString())
 		}
 		return true
 	})
 	return err
 }
+
+// setResourceContext precomputes and caches resource-level labels.
+// Called once per ResourceMetrics boundary, before processing any datapoints.
+// If an error is returned, resource level cache is reset.
+func (c *PrometheusConverter) setResourceContext(resource pcommon.Resource, settings Settings) error {
+	resourceAttrs := resource.Attributes()
+	c.resourceLabels = &cachedResourceLabels{
+		externalLabels: settings.ExternalLabels,
+	}
+
+	c.labelNamer = otlptranslator.LabelNamer{
+		UTF8Allowed:                 settings.AllowUTF8,
+		UnderscoreLabelSanitization: settings.LabelNameUnderscoreSanitization,
+		PreserveMultipleUnderscores: settings.LabelNamePreserveMultipleUnderscores,
+	}
+
+	if serviceName, ok := resourceAttrs.Get(string(semconv.ServiceNameKey)); ok {
+		val := serviceName.AsString()
+		if serviceNamespace, ok := resourceAttrs.Get(string(semconv.ServiceNamespaceKey)); ok {
+			val = serviceNamespace.AsString() + "/" + val
+		}
+		c.resourceLabels.jobLabel = val
+	}
+
+	if instance, ok := resourceAttrs.Get(string(semconv.ServiceInstanceIDKey)); ok {
+		c.resourceLabels.instanceLabel = instance.AsString()
+	}
+
+	if settings.PromoteResourceAttributes != nil {
+		c.builder.Reset(labels.EmptyLabels())
+		if err := settings.PromoteResourceAttributes.addPromotedAttributes(c.builder, resourceAttrs, c.buildLabelName); err != nil {
+			c.clearResourceContext()
+			return err
+		}
+		c.resourceLabels.promotedLabels = c.builder.Labels()
+	}
+	return nil
+}
+
+// setScopeContext precomputes and caches scope-level labels.
+// Called once per ScopeMetrics boundary, before processing any metrics.
+// If an error is returned, scope level cache is reset.
+func (c *PrometheusConverter) setScopeContext(scope scope, settings Settings) error {
+	if !settings.PromoteScopeMetadata || scope.name == "" {
+		c.scopeLabels = nil
+		return nil
+	}
+
+	c.scopeLabels = &cachedScopeLabels{
+		scopeName:      scope.name,
+		scopeVersion:   scope.version,
+		scopeSchemaURL: scope.schemaURL,
+	}
+	c.builder.Reset(labels.EmptyLabels())
+	var err error
+	scope.attributes.Range(func(k string, v pcommon.Value) bool {
+		var name string
+		name, err = c.buildLabelName("otel_scope_" + k)
+		if err != nil {
+			return false
+		}
+		c.builder.Set(name, v.AsString())
+		return true
+	})
+	if err != nil {
+		c.scopeLabels = nil
+		return err
+	}
+
+	c.scopeLabels.scopeAttrs = c.builder.Labels()
+	return nil
+}
+
+// clearResourceContext clears cached labels between ResourceMetrics.
+func (c *PrometheusConverter) clearResourceContext() {
+	c.resourceLabels = nil
+	c.scopeLabels = nil
+}

storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go

@@ -31,6 +31,7 @@ import (
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp"
 
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -456,6 +457,211 @@ func TestFromMetrics(t *testing.T) {
 			},
 		}, targetInfoSamples)
 	})
+
+	t.Run("target_info should not include scope labels when PromoteScopeMetadata is enabled", func(t *testing.T) {
+		// Regression test: When PromoteScopeMetadata is enabled and a scope has a non-empty name,
+		// the cached scopeLabels should NOT be merged into target_info.
+		request := pmetricotlp.NewExportRequest()
+		rm := request.Metrics().ResourceMetrics().AppendEmpty()
+
+		// Set up resource attributes for job/instance labels.
+		rm.Resource().Attributes().PutStr("service.name", "test-service")
+		rm.Resource().Attributes().PutStr("service.instance.id", "instance-1")
+		generateAttributes(rm.Resource().Attributes(), "resource", 2)
+
+		// Create a scope with a non-empty name (this triggers scope label caching).
+		scopeMetrics := rm.ScopeMetrics().AppendEmpty()
+		scope := scopeMetrics.Scope()
+		scope.SetName("my-scope")
+		scope.SetVersion("1.0.0")
+		scope.Attributes().PutStr("scope-attr", "scope-value")
+
+		// Add a metric.
+		ts := pcommon.NewTimestampFromTime(time.Now())
+		m := scopeMetrics.Metrics().AppendEmpty()
+		m.SetEmptyGauge()
+		m.SetName("test_gauge")
+		m.SetDescription("test gauge")
+		point := m.Gauge().DataPoints().AppendEmpty()
+		point.SetTimestamp(ts)
+		point.SetDoubleValue(1.0)
+
+		mockAppender := &mockCombinedAppender{}
+		converter := NewPrometheusConverter(mockAppender)
+		annots, err := converter.FromMetrics(
+			context.Background(),
+			request.Metrics(),
+			Settings{
+				PromoteScopeMetadata: true,
+				LookbackDelta:        defaultLookbackDelta,
+			},
+		)
+		require.NoError(t, err)
+		require.Empty(t, annots)
+		require.NoError(t, mockAppender.Commit())
+
+		// Find target_info samples.
+		var targetInfoSamples []combinedSample
+		for _, s := range mockAppender.samples {
+			if s.ls.Get(labels.MetricName) == "target_info" {
+				targetInfoSamples = append(targetInfoSamples, s)
+			}
+		}
+		require.NotEmpty(t, targetInfoSamples, "expected target_info samples")
+
+		// Verify target_info does NOT have scope labels.
+		for _, s := range targetInfoSamples {
+			require.Empty(t, s.ls.Get("otel_scope_name"), "target_info should not have otel_scope_name")
+			require.Empty(t, s.ls.Get("otel_scope_version"), "target_info should not have otel_scope_version")
+			require.Empty(t, s.ls.Get("otel_scope_schema_url"), "target_info should not have otel_scope_schema_url")
+			require.Empty(t, s.ls.Get("otel_scope_scope_attr"), "target_info should not have scope attributes")
+		}
+
+		// Verify the metric itself DOES have scope labels.
+		var metricSamples []combinedSample
+		for _, s := range mockAppender.samples {
+			if s.ls.Get(labels.MetricName) == "test_gauge" {
+				metricSamples = append(metricSamples, s)
+			}
+		}
+		require.NotEmpty(t, metricSamples, "expected metric samples")
+		require.Equal(t, "my-scope", metricSamples[0].ls.Get("otel_scope_name"), "metric should have otel_scope_name")
+		require.Equal(t, "1.0.0", metricSamples[0].ls.Get("otel_scope_version"), "metric should have otel_scope_version")
+	})
+
+	t.Run("target_info should include promoted resource attributes", func(t *testing.T) {
+		// Promoted resource attributes should appear on both metrics and target_info.
+		request := pmetricotlp.NewExportRequest()
+		rm := request.Metrics().ResourceMetrics().AppendEmpty()
+
+		// Set up resource attributes.
+		rm.Resource().Attributes().PutStr("service.name", "test-service")
+		rm.Resource().Attributes().PutStr("service.instance.id", "instance-1")
+		rm.Resource().Attributes().PutStr("custom.promoted.attr", "promoted-value")
+		rm.Resource().Attributes().PutStr("another.resource.attr", "another-value")
+
+		// Add a metric.
+		ts := pcommon.NewTimestampFromTime(time.Now())
+		scopeMetrics := rm.ScopeMetrics().AppendEmpty()
+		m := scopeMetrics.Metrics().AppendEmpty()
+		m.SetEmptyGauge()
+		m.SetName("test_gauge")
+		m.SetDescription("test gauge")
+		point := m.Gauge().DataPoints().AppendEmpty()
+		point.SetTimestamp(ts)
+		point.SetDoubleValue(1.0)
+
+		mockAppender := &mockCombinedAppender{}
+		converter := NewPrometheusConverter(mockAppender)
+		annots, err := converter.FromMetrics(
+			context.Background(),
+			request.Metrics(),
+			Settings{
+				PromoteResourceAttributes: NewPromoteResourceAttributes(config.OTLPConfig{
+					PromoteResourceAttributes: []string{"custom.promoted.attr"},
+				}),
+				LookbackDelta: defaultLookbackDelta,
+			},
+		)
+		require.NoError(t, err)
+		require.Empty(t, annots)
+		require.NoError(t, mockAppender.Commit())
+
+		// Find target_info samples.
+		var targetInfoSamples []combinedSample
+		for _, s := range mockAppender.samples {
+			if s.ls.Get(labels.MetricName) == "target_info" {
+				targetInfoSamples = append(targetInfoSamples, s)
+			}
+		}
+		require.NotEmpty(t, targetInfoSamples, "expected target_info samples")
+
+		// Verify target_info has the promoted resource attribute.
+		for _, s := range targetInfoSamples {
+			require.Equal(t, "promoted-value", s.ls.Get("custom_promoted_attr"), "target_info should have promoted resource attributes")
+			require.Equal(t, "another-value", s.ls.Get("another_resource_attr"), "target_info should have non-promoted resource attributes")
+		}
+
+		// Verify the metric also has the promoted resource attribute.
+		var metricSamples []combinedSample
+		for _, s := range mockAppender.samples {
+			if s.ls.Get(labels.MetricName) == "test_gauge" {
+				metricSamples = append(metricSamples, s)
+			}
+		}
+		require.NotEmpty(t, metricSamples, "expected metric samples")
+		require.Equal(t, "promoted-value", metricSamples[0].ls.Get("custom_promoted_attr"), "metric should have promoted resource attribute")
+	})
+
+	t.Run("target_info should include promoted attributes when KeepIdentifyingResourceAttributes is enabled", func(t *testing.T) {
+		// When both PromoteResourceAttributes and KeepIdentifyingResourceAttributes are configured,
+		// target_info should include both the promoted attributes and the identifying attributes.
+		request := pmetricotlp.NewExportRequest()
+		rm := request.Metrics().ResourceMetrics().AppendEmpty()
+
+		rm.Resource().Attributes().PutStr("service.name", "test-service")
+		rm.Resource().Attributes().PutStr("service.namespace", "test-namespace")
+		rm.Resource().Attributes().PutStr("service.instance.id", "instance-1")
+		rm.Resource().Attributes().PutStr("custom.promoted.attr", "promoted-value")
+		rm.Resource().Attributes().PutStr("another.resource.attr", "another-value")
+
+		// Add a metric.
+		ts := pcommon.NewTimestampFromTime(time.Now())
+		scopeMetrics := rm.ScopeMetrics().AppendEmpty()
+		m := scopeMetrics.Metrics().AppendEmpty()
+		m.SetEmptyGauge()
+		m.SetName("test_gauge")
+		m.SetDescription("test gauge")
+		point := m.Gauge().DataPoints().AppendEmpty()
+		point.SetTimestamp(ts)
+		point.SetDoubleValue(1.0)
+
+		mockAppender := &mockCombinedAppender{}
+		converter := NewPrometheusConverter(mockAppender)
+		annots, err := converter.FromMetrics(
+			context.Background(),
+			request.Metrics(),
+			Settings{
+				PromoteResourceAttributes: NewPromoteResourceAttributes(config.OTLPConfig{
+					PromoteResourceAttributes: []string{"custom.promoted.attr"},
+				}),
+				KeepIdentifyingResourceAttributes: true,
+				LookbackDelta:                     defaultLookbackDelta,
+			},
+		)
+		require.NoError(t, err)
+		require.Empty(t, annots)
+		require.NoError(t, mockAppender.Commit())
+
+		var targetInfoSamples []combinedSample
+		for _, s := range mockAppender.samples {
+			if s.ls.Get(labels.MetricName) == "target_info" {
+				targetInfoSamples = append(targetInfoSamples, s)
+			}
+		}
+		require.NotEmpty(t, targetInfoSamples, "expected target_info samples")
+
+		// Verify target_info has the promoted resource attribute.
+		for _, s := range targetInfoSamples {
+			require.Equal(t, "promoted-value", s.ls.Get("custom_promoted_attr"), "target_info should have promoted resource attributes")
+			// And it should have the identifying attributes (since KeepIdentifyingResourceAttributes is true).
+			require.Equal(t, "test-service", s.ls.Get("service_name"), "target_info should have service.name when KeepIdentifyingResourceAttributes is true")
+			require.Equal(t, "test-namespace", s.ls.Get("service_namespace"), "target_info should have service.namespace when KeepIdentifyingResourceAttributes is true")
+			require.Equal(t, "instance-1", s.ls.Get("service_instance_id"), "target_info should have service.instance.id when KeepIdentifyingResourceAttributes is true")
+			// And the non-promoted resource attribute.
+			require.Equal(t, "another-value", s.ls.Get("another_resource_attr"), "target_info should have non-promoted resource attributes")
+		}
+
+		// Verify the metric also has the promoted resource attribute.
+		var metricSamples []combinedSample
+		for _, s := range mockAppender.samples {
+			if s.ls.Get(labels.MetricName) == "test_gauge" {
+				metricSamples = append(metricSamples, s)
+			}
+		}
+		require.NotEmpty(t, metricSamples, "expected metric samples")
+		require.Equal(t, "promoted-value", metricSamples[0].ls.Get("custom_promoted_attr"), "metric should have promoted resource attribute")
+	})
 }
 
 func TestTemporality(t *testing.T) {
@@ -1067,7 +1273,7 @@ func BenchmarkPrometheusConverter_FromMetrics(b *testing.B) {
 
 											for b.Loop() {
 												app := &noOpAppender{}
-												mockAppender := NewCombinedAppender(app, noOpLogger, false, false, appMetrics)
+												mockAppender := NewCombinedAppender(app, noOpLogger, false, true, appMetrics)
 												converter := NewPrometheusConverter(mockAppender)
 												annots, err := converter.FromMetrics(context.Background(), payload.Metrics(), settings)
 												require.NoError(b, err)
@@ -1323,3 +1529,276 @@ func generateExemplars(exemplars pmetric.ExemplarSlice, count int, ts pcommon.Ti
 		e.SetTraceID(pcommon.TraceID{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f})
 	}
 }
+
+// createMultiScopeExportRequest creates an export request with multiple scopes per resource.
+// This is useful for benchmarking resource-level label caching, where cached resource labels
+// (job, instance, promoted attributes) should be computed once and reused across all scopes.
+func createMultiScopeExportRequest(
+	resourceAttributeCount int,
+	scopeCount int,
+	metricsPerScope int,
+	labelsPerMetric int,
+	scopeAttributeCount int,
+) pmetricotlp.ExportRequest {
+	request := pmetricotlp.NewExportRequest()
+	ts := pcommon.NewTimestampFromTime(time.Now())
+
+	rm := request.Metrics().ResourceMetrics().AppendEmpty()
+	generateAttributes(rm.Resource().Attributes(), "resource", resourceAttributeCount)
+
+	// Set service attributes for job/instance label generation
+	rm.Resource().Attributes().PutStr("service.name", "test-service")
+	rm.Resource().Attributes().PutStr("service.namespace", "test-namespace")
+	rm.Resource().Attributes().PutStr("service.instance.id", "instance-1")
+
+	for s := range scopeCount {
+		scopeMetrics := rm.ScopeMetrics().AppendEmpty()
+		scope := scopeMetrics.Scope()
+		scope.SetName(fmt.Sprintf("scope-%d", s))
+		scope.SetVersion("1.0.0")
+		generateAttributes(scope.Attributes(), "scope", scopeAttributeCount)
+
+		metrics := scopeMetrics.Metrics()
+		for m := range metricsPerScope {
+			metric := metrics.AppendEmpty()
+			metric.SetName(fmt.Sprintf("gauge_s%d_m%d", s, m))
+			metric.SetDescription("gauge metric")
+			metric.SetUnit("unit")
+			point := metric.SetEmptyGauge().DataPoints().AppendEmpty()
+			point.SetTimestamp(ts)
+			point.SetDoubleValue(float64(m))
+			generateAttributes(point.Attributes(), "series", labelsPerMetric)
+		}
+	}
+
+	return request
+}
+
+// createRepeatedLabelsExportRequest creates an export request where the same label names
+// appear repeatedly across many datapoints. This is useful for benchmarking the label
+// sanitization cache, which should reduce allocations when the same label names are seen multiple times.
+func createRepeatedLabelsExportRequest(
+	uniqueLabelNames int,
+	datapointCount int,
+	labelsPerDatapoint int,
+) pmetricotlp.ExportRequest {
+	request := pmetricotlp.NewExportRequest()
+	ts := pcommon.NewTimestampFromTime(time.Now())
+
+	rm := request.Metrics().ResourceMetrics().AppendEmpty()
+	rm.Resource().Attributes().PutStr("service.name", "test-service")
+	rm.Resource().Attributes().PutStr("service.instance.id", "instance-1")
+
+	metrics := rm.ScopeMetrics().AppendEmpty().Metrics()
+
+	// Pre-generate label names that will be reused.
+	labelNames := make([]string, uniqueLabelNames)
+	for i := range uniqueLabelNames {
+		labelNames[i] = fmt.Sprintf("label.name.%d", i)
+	}
+
+	for d := range datapointCount {
+		metric := metrics.AppendEmpty()
+		metric.SetName(fmt.Sprintf("gauge_%d", d))
+		metric.SetDescription("gauge metric")
+		metric.SetUnit("unit")
+		point := metric.SetEmptyGauge().DataPoints().AppendEmpty()
+		point.SetTimestamp(ts)
+		point.SetDoubleValue(float64(d))
+
+		// Add labels using the same label names (cycling through them).
+		for l := range labelsPerDatapoint {
+			labelName := labelNames[l%uniqueLabelNames]
+			point.Attributes().PutStr(labelName, fmt.Sprintf("value-%d-%d", d, l))
+		}
+	}
+
+	return request
+}
+
+// createMultiResourceExportRequest creates an export request with multiple ResourceMetrics.
+// This is useful for benchmarking the overhead of cache clearing between resources and
+// verifying that caching still helps within each resource.
+func createMultiResourceExportRequest(
+	resourceCount int,
+	resourceAttributeCount int,
+	metricsPerResource int,
+	labelsPerMetric int,
+) pmetricotlp.ExportRequest {
+	request := pmetricotlp.NewExportRequest()
+	ts := pcommon.NewTimestampFromTime(time.Now())
+
+	for r := range resourceCount {
+		rm := request.Metrics().ResourceMetrics().AppendEmpty()
+		generateAttributes(rm.Resource().Attributes(), "resource", resourceAttributeCount)
+
+		// Set unique service attributes per resource for job/instance label generation.
+		rm.Resource().Attributes().PutStr("service.name", fmt.Sprintf("service-%d", r))
+		rm.Resource().Attributes().PutStr("service.namespace", "test-namespace")
+		rm.Resource().Attributes().PutStr("service.instance.id", fmt.Sprintf("instance-%d", r))
+
+		metrics := rm.ScopeMetrics().AppendEmpty().Metrics()
+		for m := range metricsPerResource {
+			metric := metrics.AppendEmpty()
+			metric.SetName(fmt.Sprintf("gauge_r%d_m%d", r, m))
+			metric.SetDescription("gauge metric")
+			metric.SetUnit("unit")
+			point := metric.SetEmptyGauge().DataPoints().AppendEmpty()
+			point.SetTimestamp(ts)
+			point.SetDoubleValue(float64(m))
+			generateAttributes(point.Attributes(), "series", labelsPerMetric)
+		}
+	}
+
+	return request
+}
+
+// BenchmarkFromMetrics_LabelCaching_MultipleDatapointsPerResource benchmarks the resource-level
+// label caching optimization. With caching, resource labels (job, instance, promoted
+// attributes) should be computed once per ResourceMetrics and reused for all datapoints.
+func BenchmarkFromMetrics_LabelCaching_MultipleDatapointsPerResource(b *testing.B) {
+	const (
+		labelsPerMetric     = 5
+		scopeAttributeCount = 3
+	)
+	for _, resourceAttrs := range []int{5, 50} {
+		for _, scopeCount := range []int{1, 10} {
+			for _, metricsPerScope := range []int{10, 100} {
+				b.Run(fmt.Sprintf("res_attrs=%d/scopes=%d/metrics=%d", resourceAttrs, scopeCount, metricsPerScope), func(b *testing.B) {
+					settings := Settings{
+						PromoteResourceAttributes: NewPromoteResourceAttributes(config.OTLPConfig{
+							PromoteAllResourceAttributes: true,
+						}),
+					}
+					payload := createMultiScopeExportRequest(
+						resourceAttrs,
+						scopeCount,
+						metricsPerScope,
+						labelsPerMetric,
+						scopeAttributeCount,
+					)
+					appMetrics := NewCombinedAppenderMetrics(prometheus.NewRegistry())
+					noOpLogger := promslog.NewNopLogger()
+					b.ReportAllocs()
+					b.ResetTimer()
+
+					for b.Loop() {
+						app := &noOpAppender{}
+						mockAppender := NewCombinedAppender(app, noOpLogger, false, false, appMetrics)
+						converter := NewPrometheusConverter(mockAppender)
+						_, err := converter.FromMetrics(context.Background(), payload.Metrics(), settings)
+						require.NoError(b, err)
+					}
+				})
+			}
+		}
+	}
+}
+
+// BenchmarkFromMetrics_LabelCaching_RepeatedLabelNames benchmarks the label sanitization cache.
+// When the same label names appear across many datapoints, the sanitization should
+// only happen once per unique label name within a ResourceMetrics.
+func BenchmarkFromMetrics_LabelCaching_RepeatedLabelNames(b *testing.B) {
+	const labelsPerDatapoint = 20
+	for _, uniqueLabels := range []int{5, 50} {
+		for _, datapoints := range []int{100, 1000} {
+			b.Run(fmt.Sprintf("unique_labels=%d/datapoints=%d", uniqueLabels, datapoints), func(b *testing.B) {
+				settings := Settings{}
+				payload := createRepeatedLabelsExportRequest(
+					uniqueLabels,
+					datapoints,
+					labelsPerDatapoint,
+				)
+				appMetrics := NewCombinedAppenderMetrics(prometheus.NewRegistry())
+				noOpLogger := promslog.NewNopLogger()
+				b.ReportAllocs()
+				b.ResetTimer()
+
+				for b.Loop() {
+					app := &noOpAppender{}
+					mockAppender := NewCombinedAppender(app, noOpLogger, false, false, appMetrics)
+					converter := NewPrometheusConverter(mockAppender)
+					_, err := converter.FromMetrics(context.Background(), payload.Metrics(), settings)
+					require.NoError(b, err)
+				}
+			})
+		}
+	}
+}
+
+// BenchmarkFromMetrics_LabelCaching_ScopeMetadata benchmarks scope-level label caching when
+// PromoteScopeMetadata is enabled. Scope metadata labels (otel_scope_name, version, etc.)
+// should be computed once per ScopeMetrics and reused for all metrics within that scope.
+func BenchmarkFromMetrics_LabelCaching_ScopeMetadata(b *testing.B) {
+	const (
+		resourceAttributeCount = 5
+		labelsPerMetric        = 5
+	)
+	for _, scopeAttrs := range []int{0, 10} {
+		for _, metricsPerScope := range []int{10, 100} {
+			b.Run(fmt.Sprintf("scope_attrs=%d/metrics=%d", scopeAttrs, metricsPerScope), func(b *testing.B) {
+				settings := Settings{
+					PromoteScopeMetadata: true,
+				}
+				payload := createMultiScopeExportRequest(
+					resourceAttributeCount,
+					1, // single scope to isolate scope caching benefit
+					metricsPerScope,
+					labelsPerMetric,
+					scopeAttrs,
+				)
+				appMetrics := NewCombinedAppenderMetrics(prometheus.NewRegistry())
+				noOpLogger := promslog.NewNopLogger()
+				b.ReportAllocs()
+				b.ResetTimer()
+
+				for b.Loop() {
+					app := &noOpAppender{}
+					mockAppender := NewCombinedAppender(app, noOpLogger, false, false, appMetrics)
+					converter := NewPrometheusConverter(mockAppender)
+					_, err := converter.FromMetrics(context.Background(), payload.Metrics(), settings)
+					require.NoError(b, err)
+				}
+			})
+		}
+	}
+}
+
+// BenchmarkFromMetrics_LabelCaching_MultipleResources benchmarks requests with multiple
+// ResourceMetrics. The label sanitization cache is cleared between resources, so this
+// measures the overhead of cache clearing and verifies caching helps within each resource.
+func BenchmarkFromMetrics_LabelCaching_MultipleResources(b *testing.B) {
+	const (
+		resourceAttributeCount = 10
+		labelsPerMetric        = 10
+	)
+	for _, resourceCount := range []int{1, 10, 50} {
+		for _, metricsPerResource := range []int{10, 100} {
+			b.Run(fmt.Sprintf("resources=%d/metrics=%d", resourceCount, metricsPerResource), func(b *testing.B) {
+				settings := Settings{
+					PromoteResourceAttributes: NewPromoteResourceAttributes(config.OTLPConfig{
+						PromoteAllResourceAttributes: true,
+					}),
+				}
+				payload := createMultiResourceExportRequest(
+					resourceCount,
+					resourceAttributeCount,
+					metricsPerResource,
+					labelsPerMetric,
+				)
+				appMetrics := NewCombinedAppenderMetrics(prometheus.NewRegistry())
+				noOpLogger := promslog.NewNopLogger()
+				b.ReportAllocs()
+				b.ResetTimer()
+
+				for b.Loop() {
+					app := &noOpAppender{}
+					mockAppender := NewCombinedAppender(app, noOpLogger, false, false, appMetrics)
+					converter := NewPrometheusConverter(mockAppender)
+					_, err := converter.FromMetrics(context.Background(), payload.Metrics(), settings)
+					require.NoError(b, err)
+				}
+			})
+		}
+	}
+}

storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go

@@ -21,14 +21,13 @@ import (
 	"math"
 
 	"github.com/prometheus/common/model"
-	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 
 	"github.com/prometheus/prometheus/model/value"
 )
 
 func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, dataPoints pmetric.NumberDataPointSlice,
-	resource pcommon.Resource, settings Settings, scope scope, meta Metadata,
+	settings Settings, meta Metadata,
 ) error {
 	for x := 0; x < dataPoints.Len(); x++ {
 		if err := c.everyN.checkContext(ctx); err != nil {
@@ -37,9 +36,7 @@ func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, data
 
 		pt := dataPoints.At(x)
 		labels, err := c.createAttributes(
-			resource,
 			pt.Attributes(),
-			scope,
 			settings,
 			nil,
 			true,
@@ -71,7 +68,7 @@ func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, data
 }
 
 func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPoints pmetric.NumberDataPointSlice,
-	resource pcommon.Resource, settings Settings, scope scope, meta Metadata,
+	settings Settings, meta Metadata,
 ) error {
 	for x := 0; x < dataPoints.Len(); x++ {
 		if err := c.everyN.checkContext(ctx); err != nil {
@@ -80,9 +77,7 @@ func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPo
 
 		pt := dataPoints.At(x)
 		lbls, err := c.createAttributes(
-			resource,
 			pt.Attributes(),
-			scope,
 			settings,
 			nil,
 			true,

storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go

@@ -114,15 +114,19 @@ func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) {
 			metric := tt.metric()
 			mockAppender := &mockCombinedAppender{}
 			converter := NewPrometheusConverter(mockAppender)
+			settings := Settings{
+				PromoteScopeMetadata: tt.promoteScope,
+			}
+			resource := pcommon.NewResource()
+
+			// Initialize resource and scope context as FromMetrics would.
+			require.NoError(t, converter.setResourceContext(resource, settings))
+			require.NoError(t, converter.setScopeContext(tt.scope, settings))
 
 			converter.addGaugeNumberDataPoints(
 				context.Background(),
 				metric.Gauge().DataPoints(),
-				pcommon.NewResource(),
-				Settings{
-					PromoteScopeMetadata: tt.promoteScope,
-				},
-				tt.scope,
+				settings,
 				Metadata{
 					MetricFamilyName: metric.Name(),
 				},
@@ -344,15 +348,19 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
 			metric := tt.metric()
 			mockAppender := &mockCombinedAppender{}
 			converter := NewPrometheusConverter(mockAppender)
+			settings := Settings{
+				PromoteScopeMetadata: tt.promoteScope,
+			}
+			resource := pcommon.NewResource()
+
+			// Initialize resource and scope context as FromMetrics would.
+			require.NoError(t, converter.setResourceContext(resource, settings))
+			require.NoError(t, converter.setScopeContext(tt.scope, settings))
 
 			converter.addSumNumberDataPoints(
 				context.Background(),
 				metric.Sum().DataPoints(),
-				pcommon.NewResource(),
-				Settings{
-					PromoteScopeMetadata: tt.promoteScope,
-				},
-				tt.scope,
+				settings,
 				Metadata{
 					MetricFamilyName: metric.Name(),
 				},

storage/remote/storage.go

@@ -63,6 +63,8 @@ type Storage struct {
 	localStartTimeCallback startTimeCallback
 }
 
+var _ storage.Storage = &Storage{}
+
 // NewStorage returns a remote.Storage.
 func NewStorage(l *slog.Logger, reg prometheus.Registerer, stCallback startTimeCallback, walDir string, flushDeadline time.Duration, sm ReadyScrapeManager, enableTypeAndUnitLabels bool) *Storage {
 	if l == nil {
@@ -193,6 +195,11 @@ func (s *Storage) Appender(ctx context.Context) storage.Appender {
 	return s.rws.Appender(ctx)
 }
 
+// AppenderV2 implements storage.Storage.
+func (s *Storage) AppenderV2(ctx context.Context) storage.AppenderV2 {
+	return s.rws.AppenderV2(ctx)
+}
+
 // LowestSentTimestamp returns the lowest sent timestamp across all queues.
 func (s *Storage) LowestSentTimestamp() int64 {
 	return s.rws.LowestSentTimestamp()

storage/remote/write.go

@@ -238,8 +238,20 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error {
 // Appender implements storage.Storage.
 func (rws *WriteStorage) Appender(context.Context) storage.Appender {
 	return &timestampTracker{
-		writeStorage:         rws,
-		highestRecvTimestamp: rws.highestTimestamp,
+		baseTimestampTracker: baseTimestampTracker{
+			writeStorage:         rws,
+			highestRecvTimestamp: rws.highestTimestamp,
+		},
+	}
+}
+
+// AppenderV2 implements storage.Storage.
+func (rws *WriteStorage) AppenderV2(context.Context) storage.AppenderV2 {
+	return &timestampTrackerV2{
+		baseTimestampTracker: baseTimestampTracker{
+			writeStorage:         rws,
+			highestRecvTimestamp: rws.highestTimestamp,
+		},
 	}
 }
 
@@ -282,9 +294,9 @@ func (rws *WriteStorage) Close() error {
 	return nil
 }
 
-type timestampTracker struct {
-	writeStorage         *WriteStorage
-	appendOptions        *storage.AppendOptions
+type baseTimestampTracker struct {
+	writeStorage *WriteStorage
+
 	samples              int64
 	exemplars            int64
 	histograms           int64
@@ -292,6 +304,12 @@ type timestampTracker struct {
 	highestRecvTimestamp *maxTimestamp
 }
 
+type timestampTracker struct {
+	baseTimestampTracker
+
+	appendOptions *storage.AppendOptions
+}
+
 func (t *timestampTracker) SetOptions(opts *storage.AppendOptions) {
 	t.appendOptions = opts
 }
@@ -345,7 +363,7 @@ func (*timestampTracker) UpdateMetadata(storage.SeriesRef, labels.Labels, metada
 }
 
 // Commit implements storage.Appender.
-func (t *timestampTracker) Commit() error {
+func (t *baseTimestampTracker) Commit() error {
 	t.writeStorage.samplesIn.incr(t.samples + t.exemplars + t.histograms)
 
 	samplesIn.Add(float64(t.samples))
@@ -356,6 +374,25 @@ func (t *timestampTracker) Commit() error {
 }
 
 // Rollback implements storage.Appender.
-func (*timestampTracker) Rollback() error {
+func (*baseTimestampTracker) Rollback() error {
 	return nil
 }
+
+type timestampTrackerV2 struct {
+	baseTimestampTracker
+}
+
+// Append implements storage.AppenderV2.
+func (t *timestampTrackerV2) Append(ref storage.SeriesRef, _ labels.Labels, _, ts int64, _ float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) {
+	switch {
+	case fh != nil, h != nil:
+		t.histograms++
+	default:
+		t.samples++
+	}
+	if ts > t.highestTimestamp {
+		t.highestTimestamp = ts
+	}
+	t.exemplars += int64(len(opts.Exemplars))
+	return ref, nil
+}

storage/series.go

@@ -138,6 +138,11 @@ func (it *listSeriesIterator) AtT() int64 {
 	return s.T()
 }
 
+func (it *listSeriesIterator) AtST() int64 {
+	s := it.samples.Get(it.idx)
+	return s.ST()
+}
+
 func (it *listSeriesIterator) Next() chunkenc.ValueType {
 	it.idx++
 	if it.idx >= it.samples.Len() {
@@ -355,18 +360,20 @@ func (s *seriesToChunkEncoder) Iterator(it chunks.Iterator) chunks.Iterator {
 		lastType = typ
 
 		var (
-			t  int64
-			v  float64
-			h  *histogram.Histogram
-			fh *histogram.FloatHistogram
+			st, t int64
+			v     float64
+			h     *histogram.Histogram
+			fh    *histogram.FloatHistogram
 		)
 		switch typ {
 		case chunkenc.ValFloat:
 			t, v = seriesIter.At()
-			app.Append(t, v)
+			st = seriesIter.AtST()
+			app.Append(st, t, v)
 		case chunkenc.ValHistogram:
 			t, h = seriesIter.AtHistogram(nil)
-			newChk, recoded, app, err = app.AppendHistogram(nil, t, h, false)
+			st = seriesIter.AtST()
+			newChk, recoded, app, err = app.AppendHistogram(nil, st, t, h, false)
 			if err != nil {
 				return errChunksIterator{err: err}
 			}
@@ -381,7 +388,8 @@ func (s *seriesToChunkEncoder) Iterator(it chunks.Iterator) chunks.Iterator {
 			}
 		case chunkenc.ValFloatHistogram:
 			t, fh = seriesIter.AtFloatHistogram(nil)
-			newChk, recoded, app, err = app.AppendFloatHistogram(nil, t, fh, false)
+			st = seriesIter.AtST()
+			newChk, recoded, app, err = app.AppendFloatHistogram(nil, st, t, fh, false)
 			if err != nil {
 				return errChunksIterator{err: err}
 			}
@@ -440,25 +448,25 @@ func (e errChunksIterator) Err() error    { return e.err }
 // Optionally it takes samples constructor, useful when you want to compare sample slices with different
 // sample implementations. if nil, sample type from this package will be used.
 // For float sample, NaN values are replaced with -42.
-func ExpandSamples(iter chunkenc.Iterator, newSampleFn func(t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) {
+func ExpandSamples(iter chunkenc.Iterator, newSampleFn func(st, t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) {
 	return expandSamples(iter, true, newSampleFn)
 }
 
 // ExpandSamplesWithoutReplacingNaNs is same as ExpandSamples but it does not replace float sample NaN values with anything.
-func ExpandSamplesWithoutReplacingNaNs(iter chunkenc.Iterator, newSampleFn func(t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) {
+func ExpandSamplesWithoutReplacingNaNs(iter chunkenc.Iterator, newSampleFn func(st, t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) {
 	return expandSamples(iter, false, newSampleFn)
 }
 
-func expandSamples(iter chunkenc.Iterator, replaceNaN bool, newSampleFn func(t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) {
+func expandSamples(iter chunkenc.Iterator, replaceNaN bool, newSampleFn func(st, t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample) ([]chunks.Sample, error) {
 	if newSampleFn == nil {
-		newSampleFn = func(t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample {
+		newSampleFn = func(st, t int64, f float64, h *histogram.Histogram, fh *histogram.FloatHistogram) chunks.Sample {
 			switch {
 			case h != nil:
-				return hSample{t, h}
+				return hSample{st, t, h}
 			case fh != nil:
-				return fhSample{t, fh}
+				return fhSample{st, t, fh}
 			default:
-				return fSample{t, f}
+				return fSample{st, t, f}
 			}
 		}
 	}
@@ -470,17 +478,20 @@ func expandSamples(iter chunkenc.Iterator, replaceNaN bool, newSampleFn func(t i
 			return result, iter.Err()
 		case chunkenc.ValFloat:
 			t, f := iter.At()
+			st := iter.AtST()
 			// NaNs can't be compared normally, so substitute for another value.
 			if replaceNaN && math.IsNaN(f) {
 				f = -42
 			}
-			result = append(result, newSampleFn(t, f, nil, nil))
+			result = append(result, newSampleFn(st, t, f, nil, nil))
 		case chunkenc.ValHistogram:
 			t, h := iter.AtHistogram(nil)
-			result = append(result, newSampleFn(t, 0, h, nil))
+			st := iter.AtST()
+			result = append(result, newSampleFn(st, t, 0, h, nil))
 		case chunkenc.ValFloatHistogram:
 			t, fh := iter.AtFloatHistogram(nil)
-			result = append(result, newSampleFn(t, 0, nil, fh))
+			st := iter.AtST()
+			result = append(result, newSampleFn(st, t, 0, nil, fh))
 		}
 	}
 }

storage/series_test.go

@@ -28,11 +28,11 @@ import (
 
 func TestListSeriesIterator(t *testing.T) {
 	it := NewListSeriesIterator(samples{
-		fSample{0, 0},
-		fSample{1, 1},
-		fSample{1, 1.5},
-		fSample{2, 2},
-		fSample{3, 3},
+		fSample{-10, 0, 0},
+		fSample{-9, 1, 1},
+		fSample{-8, 1, 1.5},
+		fSample{-7, 2, 2},
+		fSample{-6, 3, 3},
 	})
 
 	// Seek to the first sample with ts=1.
@@ -40,30 +40,35 @@ func TestListSeriesIterator(t *testing.T) {
 	ts, v := it.At()
 	require.Equal(t, int64(1), ts)
 	require.Equal(t, 1., v)
+	require.Equal(t, int64(-9), it.AtST())
 
 	// Seek one further, next sample still has ts=1.
 	require.Equal(t, chunkenc.ValFloat, it.Next())
 	ts, v = it.At()
 	require.Equal(t, int64(1), ts)
 	require.Equal(t, 1.5, v)
+	require.Equal(t, int64(-8), it.AtST())
 
 	// Seek again to 1 and make sure we stay where we are.
 	require.Equal(t, chunkenc.ValFloat, it.Seek(1))
 	ts, v = it.At()
 	require.Equal(t, int64(1), ts)
 	require.Equal(t, 1.5, v)
+	require.Equal(t, int64(-8), it.AtST())
 
 	// Another seek.
 	require.Equal(t, chunkenc.ValFloat, it.Seek(3))
 	ts, v = it.At()
 	require.Equal(t, int64(3), ts)
 	require.Equal(t, 3., v)
+	require.Equal(t, int64(-6), it.AtST())
 
 	// And we don't go back.
 	require.Equal(t, chunkenc.ValFloat, it.Seek(2))
 	ts, v = it.At()
 	require.Equal(t, int64(3), ts)
 	require.Equal(t, 3., v)
+	require.Equal(t, int64(-6), it.AtST())
 
 	// Seek beyond the end.
 	require.Equal(t, chunkenc.ValNone, it.Seek(5))

tsdb/agent/db.go

@@ -37,7 +37,6 @@ import (
 	"github.com/prometheus/prometheus/storage/remote"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunks"
-	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 	"github.com/prometheus/prometheus/tsdb/record"
 	"github.com/prometheus/prometheus/tsdb/tsdbutil"
 	"github.com/prometheus/prometheus/tsdb/wlog"
@@ -798,7 +797,7 @@ func (db *DB) Close() error {
 
 	db.metrics.Unregister()
 
-	return tsdb_errors.NewMulti(db.locker.Release(), db.wal.Close()).Err()
+	return errors.Join(db.locker.Release(), db.wal.Close())
 }
 
 type appenderBase struct {

tsdb/block_test.go

@@ -176,7 +176,7 @@ func TestCorruptedChunk(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			tmpdir := t.TempDir()
 
-			series := storage.NewListSeries(labels.FromStrings("a", "b"), []chunks.Sample{sample{1, 1, nil, nil}})
+			series := storage.NewListSeries(labels.FromStrings("a", "b"), []chunks.Sample{sample{0, 1, 1, nil, nil}})
 			blockDir := createBlock(t, tmpdir, []storage.Series{series})
 			files, err := sequenceFiles(chunkDir(blockDir))
 			require.NoError(t, err)
@@ -236,7 +236,7 @@ func TestLabelValuesWithMatchers(t *testing.T) {
 		seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings(
 			"tens", fmt.Sprintf("value%d", i/10),
 			"unique", fmt.Sprintf("value%d", i),
-		), []chunks.Sample{sample{100, 0, nil, nil}}))
+		), []chunks.Sample{sample{0, 100, 0, nil, nil}}))
 	}
 
 	blockDir := createBlock(t, tmpdir, seriesEntries)
@@ -319,7 +319,7 @@ func TestBlockQuerierReturnsSortedLabelValues(t *testing.T) {
 	for i := 100; i > 0; i-- {
 		seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings(
 			"__name__", fmt.Sprintf("value%d", i),
-		), []chunks.Sample{sample{100, 0, nil, nil}}))
+		), []chunks.Sample{sample{0, 100, 0, nil, nil}}))
 	}
 
 	blockDir := createBlock(t, tmpdir, seriesEntries)
@@ -436,7 +436,7 @@ func BenchmarkLabelValuesWithMatchers(b *testing.B) {
 			"a_unique", fmt.Sprintf("value%d", i),
 			"b_tens", fmt.Sprintf("value%d", i/(metricCount/10)),
 			"c_ninety", fmt.Sprintf("value%d", i/(metricCount/10)/9), // "0" for the first 90%, then "1"
-		), []chunks.Sample{sample{100, 0, nil, nil}}))
+		), []chunks.Sample{sample{0, 100, 0, nil, nil}}))
 	}
 
 	blockDir := createBlock(b, tmpdir, seriesEntries)
@@ -472,13 +472,13 @@ func TestLabelNamesWithMatchers(t *testing.T) {
 	for i := range 100 {
 		seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings(
 			"unique", fmt.Sprintf("value%d", i),
-		), []chunks.Sample{sample{100, 0, nil, nil}}))
+		), []chunks.Sample{sample{0, 100, 0, nil, nil}}))
 
 		if i%10 == 0 {
 			seriesEntries = append(seriesEntries, storage.NewListSeries(labels.FromStrings(
 				"tens", fmt.Sprintf("value%d", i/10),
 				"unique", fmt.Sprintf("value%d", i),
-			), []chunks.Sample{sample{100, 0, nil, nil}}))
+			), []chunks.Sample{sample{0, 100, 0, nil, nil}}))
 		}
 
 		if i%20 == 0 {
@@ -486,7 +486,7 @@ func TestLabelNamesWithMatchers(t *testing.T) {
 				"tens", fmt.Sprintf("value%d", i/10),
 				"twenties", fmt.Sprintf("value%d", i/20),
 				"unique", fmt.Sprintf("value%d", i),
-			), []chunks.Sample{sample{100, 0, nil, nil}}))
+			), []chunks.Sample{sample{0, 100, 0, nil, nil}}))
 		}
 	}
 
@@ -542,7 +542,7 @@ func TestBlockIndexReader_PostingsForLabelMatching(t *testing.T) {
 	testPostingsForLabelMatching(t, 2, func(t *testing.T, series []labels.Labels) IndexReader {
 		var seriesEntries []storage.Series
 		for _, s := range series {
-			seriesEntries = append(seriesEntries, storage.NewListSeries(s, []chunks.Sample{sample{100, 0, nil, nil}}))
+			seriesEntries = append(seriesEntries, storage.NewListSeries(s, []chunks.Sample{sample{0, 100, 0, nil, nil}}))
 		}
 
 		blockDir := createBlock(t, t.TempDir(), seriesEntries)

tsdb/chunkenc/chunk.go

@@ -99,9 +99,9 @@ type Iterable interface {
 	Iterator(Iterator) Iterator
 }
 
-// Appender adds sample pairs to a chunk.
+// Appender adds sample with start timestamp, timestamp, and value to a chunk.
 type Appender interface {
-	Append(int64, float64)
+	Append(st, t int64, v float64)
 
 	// AppendHistogram and AppendFloatHistogram append a histogram sample to a histogram or float histogram chunk.
 	// Appending a histogram may require creating a completely new chunk or recoding (changing) the current chunk.
@@ -114,8 +114,8 @@ type Appender interface {
 	// The returned bool isRecoded can be used to distinguish between the new Chunk c being a completely new Chunk
 	// or the current Chunk recoded to a new Chunk.
 	// The Appender app that can be used for the next append is always returned.
-	AppendHistogram(prev *HistogramAppender, t int64, h *histogram.Histogram, appendOnly bool) (c Chunk, isRecoded bool, app Appender, err error)
-	AppendFloatHistogram(prev *FloatHistogramAppender, t int64, h *histogram.FloatHistogram, appendOnly bool) (c Chunk, isRecoded bool, app Appender, err error)
+	AppendHistogram(prev *HistogramAppender, st, t int64, h *histogram.Histogram, appendOnly bool) (c Chunk, isRecoded bool, app Appender, err error)
+	AppendFloatHistogram(prev *FloatHistogramAppender, st, t int64, h *histogram.FloatHistogram, appendOnly bool) (c Chunk, isRecoded bool, app Appender, err error)
 }
 
 // Iterator is a simple iterator that can only get the next value.
@@ -151,6 +151,10 @@ type Iterator interface {
 	// AtT returns the current timestamp.
 	// Before the iterator has advanced, the behaviour is unspecified.
 	AtT() int64
+	// AtST returns the current start timestamp.
+	// Returns 0 if the start timestamp is not implemented or not set.
+	// Before the iterator has advanced, the behaviour is unspecified.
+	AtST() int64
 	// Err returns the current error. It should be used only after the
 	// iterator is exhausted, i.e. `Next` or `Seek` have returned ValNone.
 	Err() error
@@ -208,25 +212,30 @@ func (v ValueType) NewChunk() (Chunk, error) {
 	}
 }
 
-// MockSeriesIterator returns an iterator for a mock series with custom timeStamps and values.
-func MockSeriesIterator(timestamps []int64, values []float64) Iterator {
+// MockSeriesIterator returns an iterator for a mock series with custom
+// start timestamp, timestamps, and values.
+// Start timestamps is optional, pass nil or empty slice to indicate no start
+// timestamps.
+func MockSeriesIterator(startTimestamps, timestamps []int64, values []float64) Iterator {
 	return &mockSeriesIterator{
-		timeStamps: timestamps,
-		values:     values,
-		currIndex:  -1,
+		startTimestamps: startTimestamps,
+		timestamps:      timestamps,
+		values:          values,
+		currIndex:       -1,
 	}
 }
 
 type mockSeriesIterator struct {
-	timeStamps []int64
-	values     []float64
-	currIndex  int
+	timestamps      []int64
+	startTimestamps []int64
+	values          []float64
+	currIndex       int
 }
 
 func (*mockSeriesIterator) Seek(int64) ValueType { return ValNone }
 
 func (it *mockSeriesIterator) At() (int64, float64) {
-	return it.timeStamps[it.currIndex], it.values[it.currIndex]
+	return it.timestamps[it.currIndex], it.values[it.currIndex]
 }
 
 func (*mockSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
@@ -238,11 +247,18 @@ func (*mockSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *
 }
 
 func (it *mockSeriesIterator) AtT() int64 {
-	return it.timeStamps[it.currIndex]
+	return it.timestamps[it.currIndex]
+}
+
+func (it *mockSeriesIterator) AtST() int64 {
+	if len(it.startTimestamps) == 0 {
+		return 0
+	}
+	return it.startTimestamps[it.currIndex]
 }
 
 func (it *mockSeriesIterator) Next() ValueType {
-	if it.currIndex < len(it.timeStamps)-1 {
+	if it.currIndex < len(it.timestamps)-1 {
 		it.currIndex++
 		return ValFloat
 	}
@@ -268,8 +284,9 @@ func (nopIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogra
 func (nopIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
 	return math.MinInt64, nil
 }
-func (nopIterator) AtT() int64 { return math.MinInt64 }
-func (nopIterator) Err() error { return nil }
+func (nopIterator) AtT() int64  { return math.MinInt64 }
+func (nopIterator) AtST() int64 { return 0 }
+func (nopIterator) Err() error  { return nil }
 
 // Pool is used to create and reuse chunk references to avoid allocations.
 type Pool interface {

tsdb/chunkenc/chunk_test.go

@@ -65,7 +65,7 @@ func testChunk(t *testing.T, c Chunk) {
 			require.NoError(t, err)
 		}
 
-		app.Append(ts, v)
+		app.Append(0, ts, v)
 		exp = append(exp, pair{t: ts, v: v})
 	}
 
@@ -226,7 +226,7 @@ func benchmarkIterator(b *testing.B, newChunk func() Chunk) {
 			if j > 250 {
 				break
 			}
-			a.Append(p.t, p.v)
+			a.Append(0, p.t, p.v)
 			j++
 		}
 	}
@@ -303,7 +303,7 @@ func benchmarkAppender(b *testing.B, deltas func() (int64, float64), newChunk fu
 			b.Fatalf("get appender: %s", err)
 		}
 		for _, p := range exp {
-			a.Append(p.t, p.v)
+			a.Append(0, p.t, p.v)
 		}
 	}
 }

tsdb/chunkenc/float_histogram.go

@@ -195,7 +195,7 @@ func (a *FloatHistogramAppender) NumSamples() int {
 
 // Append implements Appender. This implementation panics because normal float
 // samples must never be appended to a histogram chunk.
-func (*FloatHistogramAppender) Append(int64, float64) {
+func (*FloatHistogramAppender) Append(int64, int64, float64) {
 	panic("appended a float sample to a histogram chunk")
 }
 
@@ -682,11 +682,11 @@ func (*FloatHistogramAppender) recodeHistogram(
 	}
 }
 
-func (*FloatHistogramAppender) AppendHistogram(*HistogramAppender, int64, *histogram.Histogram, bool) (Chunk, bool, Appender, error) {
+func (*FloatHistogramAppender) AppendHistogram(*HistogramAppender, int64, int64, *histogram.Histogram, bool) (Chunk, bool, Appender, error) {
 	panic("appended a histogram sample to a float histogram chunk")
 }
 
-func (a *FloatHistogramAppender) AppendFloatHistogram(prev *FloatHistogramAppender, t int64, h *histogram.FloatHistogram, appendOnly bool) (Chunk, bool, Appender, error) {
+func (a *FloatHistogramAppender) AppendFloatHistogram(prev *FloatHistogramAppender, _, t int64, h *histogram.FloatHistogram, appendOnly bool) (Chunk, bool, Appender, error) {
 	if a.NumSamples() == 0 {
 		a.appendFloatHistogram(t, h)
 		if h.CounterResetHint == histogram.GaugeType {
@@ -938,6 +938,10 @@ func (it *floatHistogramIterator) AtT() int64 {
 	return it.t
 }
 
+func (*floatHistogramIterator) AtST() int64 {
+	return 0
+}
+
 func (it *floatHistogramIterator) Err() error {
 	return it.err
 }

tsdb/chunkenc/float_histogram_test.go

@@ -63,7 +63,7 @@ func TestFirstFloatHistogramExplicitCounterReset(t *testing.T) {
 			chk := NewFloatHistogramChunk()
 			app, err := chk.Appender()
 			require.NoError(t, err)
-			newChk, recoded, newApp, err := app.AppendFloatHistogram(nil, 0, h, false)
+			newChk, recoded, newApp, err := app.AppendFloatHistogram(nil, 0, 0, h, false)
 			require.NoError(t, err)
 			require.Nil(t, newChk)
 			require.False(t, recoded)
@@ -101,7 +101,7 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 		},
 		NegativeBuckets: []int64{2, 1, -1, -1}, // counts: 2, 3, 2, 1 (total 8)
 	}
-	chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.ToFloat(nil), false)
+	chk, _, app, err := app.AppendFloatHistogram(nil, 0, ts, h.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	exp = append(exp, floatResult{t: ts, h: h.ToFloat(nil)})
@@ -115,7 +115,7 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 	h.Sum = 24.4
 	h.PositiveBuckets = []int64{5, -2, 1, -2} // counts: 5, 3, 4, 2 (total 14)
 	h.NegativeBuckets = []int64{4, -1, 1, -1} // counts: 4, 3, 4, 4 (total 15)
-	chk, _, _, err = app.AppendFloatHistogram(nil, ts, h.ToFloat(nil), false)
+	chk, _, _, err = app.AppendFloatHistogram(nil, 0, ts, h.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	expH := h.ToFloat(nil)
@@ -134,7 +134,7 @@ func TestFloatHistogramChunkSameBuckets(t *testing.T) {
 	h.Sum = 24.4
 	h.PositiveBuckets = []int64{6, 1, -3, 6} // counts: 6, 7, 4, 10 (total 27)
 	h.NegativeBuckets = []int64{5, 1, -2, 3} // counts: 5, 6, 4, 7 (total 22)
-	chk, _, _, err = app.AppendFloatHistogram(nil, ts, h.ToFloat(nil), false)
+	chk, _, _, err = app.AppendFloatHistogram(nil, 0, ts, h.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	expH = h.ToFloat(nil)
@@ -224,7 +224,7 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) {
 		NegativeBuckets: []int64{1},
 	}
 
-	chk, _, app, err := app.AppendFloatHistogram(nil, ts1, h1.ToFloat(nil), false)
+	chk, _, app, err := app.AppendFloatHistogram(nil, 0, ts1, h1.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	require.Equal(t, 1, c.NumSamples())
@@ -260,7 +260,7 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) {
 	require.True(t, ok) // Only new buckets came in.
 	require.False(t, cr)
 	c, app = hApp.recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans)
-	chk, _, _, err = app.AppendFloatHistogram(nil, ts2, h2.ToFloat(nil), false)
+	chk, _, _, err = app.AppendFloatHistogram(nil, 0, ts2, h2.ToFloat(nil), false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	require.Equal(t, 2, c.NumSamples())
@@ -330,7 +330,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 
 		ts := int64(1234567890)
 
-		chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.Copy(), false)
+		chk, _, app, err := app.AppendFloatHistogram(nil, 0, ts, h.Copy(), false)
 		require.NoError(t, err)
 		require.Nil(t, chk)
 		require.Equal(t, 1, c.NumSamples())
@@ -557,7 +557,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 		nextChunk := NewFloatHistogramChunk()
 		app, err := nextChunk.Appender()
 		require.NoError(t, err)
-		newChunk, recoded, newApp, err := app.AppendFloatHistogram(hApp, ts+1, h2, false)
+		newChunk, recoded, newApp, err := app.AppendFloatHistogram(hApp, 0, ts+1, h2, false)
 		require.NoError(t, err)
 		require.Nil(t, newChunk)
 		require.False(t, recoded)
@@ -575,7 +575,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 		nextChunk := NewFloatHistogramChunk()
 		app, err := nextChunk.Appender()
 		require.NoError(t, err)
-		newChunk, recoded, newApp, err := app.AppendFloatHistogram(hApp, ts+1, h2, false)
+		newChunk, recoded, newApp, err := app.AppendFloatHistogram(hApp, 0, ts+1, h2, false)
 		require.NoError(t, err)
 		require.Nil(t, newChunk)
 		require.False(t, recoded)
@@ -602,7 +602,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 		nextChunk := NewFloatHistogramChunk()
 		app, err := nextChunk.Appender()
 		require.NoError(t, err)
-		newChunk, recoded, newApp, err := app.AppendFloatHistogram(hApp, ts+1, h2, false)
+		newChunk, recoded, newApp, err := app.AppendFloatHistogram(hApp, 0, ts+1, h2, false)
 		require.NoError(t, err)
 		require.Nil(t, newChunk)
 		require.False(t, recoded)
@@ -717,7 +717,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
 
 func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader, expectHint histogram.CounterResetHint) {
 	oldChunkBytes := oldChunk.Bytes()
-	newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, ts, h, false)
+	newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, 0, ts, h, false)
 	require.Equal(t, oldChunkBytes, oldChunk.Bytes()) // Sanity check that previous chunk is untouched.
 	require.NoError(t, err)
 	require.NotNil(t, newChunk)
@@ -732,7 +732,7 @@ func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Fl
 
 func assertNoNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) {
 	oldChunkBytes := oldChunk.Bytes()
-	newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, ts, h, false)
+	newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, 0, ts, h, false)
 	require.Greater(t, len(oldChunk.Bytes()), len(oldChunkBytes)) // Check that current chunk is bigger than previously.
 	require.NoError(t, err)
 	require.Nil(t, newChunk)
@@ -745,7 +745,7 @@ func assertNoNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *
 
 func assertRecodedFloatHistogramChunkOnAppend(t *testing.T, prevChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) {
 	prevChunkBytes := prevChunk.Bytes()
-	newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, ts, h, false)
+	newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, 0, ts, h, false)
 	require.Equal(t, prevChunkBytes, prevChunk.Bytes()) // Sanity check that previous chunk is untouched. This may change in the future if we implement in-place recoding.
 	require.NoError(t, err)
 	require.NotNil(t, newChunk)
@@ -959,7 +959,7 @@ func TestFloatHistogramChunkAppendableWithEmptySpan(t *testing.T) {
 			require.NoError(t, err)
 			require.Equal(t, 0, c.NumSamples())
 
-			_, _, _, err = app.AppendFloatHistogram(nil, 1, tc.h1, true)
+			_, _, _, err = app.AppendFloatHistogram(nil, 0, 1, tc.h1, true)
 			require.NoError(t, err)
 			require.Equal(t, 1, c.NumSamples())
 			hApp, _ := app.(*FloatHistogramAppender)
@@ -1019,7 +1019,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
 
 		ts := int64(1234567890)
 
-		chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.Copy(), false)
+		chk, _, app, err := app.AppendFloatHistogram(nil, 0, ts, h.Copy(), false)
 		require.NoError(t, err)
 		require.Nil(t, chk)
 		require.Equal(t, 1, c.NumSamples())
@@ -1259,7 +1259,7 @@ func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
 
 		h := tsdbutil.GenerateTestFloatHistogram(0)
 		var isRecoded bool
-		c, isRecoded, app, err = app.AppendFloatHistogram(nil, 1, h, true)
+		c, isRecoded, app, err = app.AppendFloatHistogram(nil, 0, 1, h, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.NoError(t, err)
@@ -1267,7 +1267,7 @@ func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
 		// Add erroring histogram.
 		h2 := h.Copy()
 		h2.Schema++
-		c, isRecoded, _, err = app.AppendFloatHistogram(nil, 2, h2, true)
+		c, isRecoded, _, err = app.AppendFloatHistogram(nil, 0, 2, h2, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.EqualError(t, err, "float histogram schema change")
@@ -1281,7 +1281,7 @@ func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
 
 		h := tsdbutil.GenerateTestFloatHistogram(0)
 		var isRecoded bool
-		c, isRecoded, app, err = app.AppendFloatHistogram(nil, 1, h, true)
+		c, isRecoded, app, err = app.AppendFloatHistogram(nil, 0, 1, h, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.NoError(t, err)
@@ -1289,7 +1289,7 @@ func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
 		// Add erroring histogram.
 		h2 := h.Copy()
 		h2.CounterResetHint = histogram.CounterReset
-		c, isRecoded, _, err = app.AppendFloatHistogram(nil, 2, h2, true)
+		c, isRecoded, _, err = app.AppendFloatHistogram(nil, 0, 2, h2, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.EqualError(t, err, "float histogram counter reset")
@@ -1303,7 +1303,7 @@ func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
 
 		h := tsdbutil.GenerateTestCustomBucketsFloatHistogram(0)
 		var isRecoded bool
-		c, isRecoded, app, err = app.AppendFloatHistogram(nil, 1, h, true)
+		c, isRecoded, app, err = app.AppendFloatHistogram(nil, 0, 1, h, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.NoError(t, err)
@@ -1311,7 +1311,7 @@ func TestFloatHistogramAppendOnlyErrors(t *testing.T) {
 		// Add erroring histogram.
 		h2 := h.Copy()
 		h2.CustomValues = []float64{0, 1, 2, 3, 4, 5, 6, 7}
-		c, isRecoded, _, err = app.AppendFloatHistogram(nil, 2, h2, true)
+		c, isRecoded, _, err = app.AppendFloatHistogram(nil, 0, 2, h2, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.EqualError(t, err, "float histogram counter reset")
@@ -1344,10 +1344,10 @@ func TestFloatHistogramUniqueSpansAfterNext(t *testing.T) {
 	app, err := c.Appender()
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendFloatHistogram(nil, 0, h1, false)
+	_, _, _, err = app.AppendFloatHistogram(nil, 0, 0, h1, false)
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendFloatHistogram(nil, 1, h2, false)
+	_, _, _, err = app.AppendFloatHistogram(nil, 0, 1, h2, false)
 	require.NoError(t, err)
 
 	// Create an iterator and advance to the first histogram.
@@ -1390,10 +1390,10 @@ func TestFloatHistogramUniqueCustomValuesAfterNext(t *testing.T) {
 	app, err := c.Appender()
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendFloatHistogram(nil, 0, h1, false)
+	_, _, _, err = app.AppendFloatHistogram(nil, 0, 0, h1, false)
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendFloatHistogram(nil, 1, h2, false)
+	_, _, _, err = app.AppendFloatHistogram(nil, 0, 1, h2, false)
 	require.NoError(t, err)
 
 	// Create an iterator and advance to the first histogram.
@@ -1435,7 +1435,7 @@ func TestFloatHistogramEmptyBucketsWithGaps(t *testing.T) {
 	c := NewFloatHistogramChunk()
 	app, err := c.Appender()
 	require.NoError(t, err)
-	_, _, _, err = app.AppendFloatHistogram(nil, 1, h1, false)
+	_, _, _, err = app.AppendFloatHistogram(nil, 0, 1, h1, false)
 	require.NoError(t, err)
 
 	h2 := &histogram.FloatHistogram{
@@ -1448,7 +1448,7 @@ func TestFloatHistogramEmptyBucketsWithGaps(t *testing.T) {
 	}
 	require.NoError(t, h2.Validate())
 
-	newC, recoded, _, err := app.AppendFloatHistogram(nil, 2, h2, false)
+	newC, recoded, _, err := app.AppendFloatHistogram(nil, 0, 2, h2, false)
 	require.NoError(t, err)
 	require.True(t, recoded)
 	require.NotNil(t, newC)
@@ -1483,7 +1483,7 @@ func TestFloatHistogramIteratorFailIfSchemaInValid(t *testing.T) {
 			app, err := c.Appender()
 			require.NoError(t, err)
 
-			_, _, _, err = app.AppendFloatHistogram(nil, 1, h, false)
+			_, _, _, err = app.AppendFloatHistogram(nil, 0, 1, h, false)
 			require.NoError(t, err)
 
 			it := c.Iterator(nil)
@@ -1512,7 +1512,7 @@ func TestFloatHistogramIteratorReduceSchema(t *testing.T) {
 			app, err := c.Appender()
 			require.NoError(t, err)
 
-			_, _, _, err = app.AppendFloatHistogram(nil, 1, h, false)
+			_, _, _, err = app.AppendFloatHistogram(nil, 0, 1, h, false)
 			require.NoError(t, err)
 
 			it := c.Iterator(nil)

tsdb/chunkenc/histogram.go

@@ -219,7 +219,7 @@ func (a *HistogramAppender) NumSamples() int {
 
 // Append implements Appender. This implementation panics because normal float
 // samples must never be appended to a histogram chunk.
-func (*HistogramAppender) Append(int64, float64) {
+func (*HistogramAppender) Append(int64, int64, float64) {
 	panic("appended a float sample to a histogram chunk")
 }
 
@@ -734,11 +734,11 @@ func (a *HistogramAppender) writeSumDelta(v float64) {
 	xorWrite(a.b, v, a.sum, &a.leading, &a.trailing)
 }
 
-func (*HistogramAppender) AppendFloatHistogram(*FloatHistogramAppender, int64, *histogram.FloatHistogram, bool) (Chunk, bool, Appender, error) {
+func (*HistogramAppender) AppendFloatHistogram(*FloatHistogramAppender, int64, int64, *histogram.FloatHistogram, bool) (Chunk, bool, Appender, error) {
 	panic("appended a float histogram sample to a histogram chunk")
 }
 
-func (a *HistogramAppender) AppendHistogram(prev *HistogramAppender, t int64, h *histogram.Histogram, appendOnly bool) (Chunk, bool, Appender, error) {
+func (a *HistogramAppender) AppendHistogram(prev *HistogramAppender, _, t int64, h *histogram.Histogram, appendOnly bool) (Chunk, bool, Appender, error) {
 	if a.NumSamples() == 0 {
 		a.appendHistogram(t, h)
 		if h.CounterResetHint == histogram.GaugeType {
@@ -1075,6 +1075,10 @@ func (it *histogramIterator) AtT() int64 {
 	return it.t
 }
 
+func (*histogramIterator) AtST() int64 {
+	return 0
+}
+
 func (it *histogramIterator) Err() error {
 	return it.err
 }

tsdb/chunkenc/histogram_test.go

@@ -64,7 +64,7 @@ func TestFirstHistogramExplicitCounterReset(t *testing.T) {
 			chk := NewHistogramChunk()
 			app, err := chk.Appender()
 			require.NoError(t, err)
-			newChk, recoded, newApp, err := app.AppendHistogram(nil, 0, h, false)
+			newChk, recoded, newApp, err := app.AppendHistogram(nil, 0, 0, h, false)
 			require.NoError(t, err)
 			require.Nil(t, newChk)
 			require.False(t, recoded)
@@ -102,7 +102,7 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 		},
 		NegativeBuckets: []int64{2, 1, -1, -1}, // counts: 2, 3, 2, 1 (total 8)
 	}
-	chk, _, app, err := app.AppendHistogram(nil, ts, h, false)
+	chk, _, app, err := app.AppendHistogram(nil, 0, ts, h, false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	exp = append(exp, result{t: ts, h: h, fh: h.ToFloat(nil)})
@@ -116,7 +116,7 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 	h.Sum = 24.4
 	h.PositiveBuckets = []int64{5, -2, 1, -2} // counts: 5, 3, 4, 2 (total 14)
 	h.NegativeBuckets = []int64{4, -1, 1, -1} // counts: 4, 3, 4, 4 (total 15)
-	chk, _, _, err = app.AppendHistogram(nil, ts, h, false)
+	chk, _, _, err = app.AppendHistogram(nil, 0, ts, h, false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	hExp := h.Copy()
@@ -135,7 +135,7 @@ func TestHistogramChunkSameBuckets(t *testing.T) {
 	h.Sum = 24.4
 	h.PositiveBuckets = []int64{6, 1, -3, 6} // counts: 6, 7, 4, 10 (total 27)
 	h.NegativeBuckets = []int64{5, 1, -2, 3} // counts: 5, 6, 4, 7 (total 22)
-	chk, _, _, err = app.AppendHistogram(nil, ts, h, false)
+	chk, _, _, err = app.AppendHistogram(nil, 0, ts, h, false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	hExp = h.Copy()
@@ -235,7 +235,7 @@ func TestHistogramChunkBucketChanges(t *testing.T) {
 		NegativeBuckets: []int64{1},
 	}
 
-	chk, _, app, err := app.AppendHistogram(nil, ts1, h1, false)
+	chk, _, app, err := app.AppendHistogram(nil, 0, ts1, h1, false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 	require.Equal(t, 1, c.NumSamples())
@@ -271,7 +271,7 @@ func TestHistogramChunkBucketChanges(t *testing.T) {
 	require.True(t, ok) // Only new buckets came in.
 	require.Equal(t, NotCounterReset, cr)
 	c, app = hApp.recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans)
-	chk, _, _, err = app.AppendHistogram(nil, ts2, h2, false)
+	chk, _, _, err = app.AppendHistogram(nil, 0, ts2, h2, false)
 	require.NoError(t, err)
 	require.Nil(t, chk)
 
@@ -344,7 +344,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 
 		ts := int64(1234567890)
 
-		chk, _, app, err := app.AppendHistogram(nil, ts, h.Copy(), false)
+		chk, _, app, err := app.AppendHistogram(nil, 0, ts, h.Copy(), false)
 		require.NoError(t, err)
 		require.Nil(t, chk)
 		require.Equal(t, 1, c.NumSamples())
@@ -581,7 +581,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 		nextChunk := NewHistogramChunk()
 		app, err := nextChunk.Appender()
 		require.NoError(t, err)
-		newChunk, recoded, newApp, err := app.AppendHistogram(hApp, ts+1, h2, false)
+		newChunk, recoded, newApp, err := app.AppendHistogram(hApp, 0, ts+1, h2, false)
 		require.NoError(t, err)
 		require.Nil(t, newChunk)
 		require.False(t, recoded)
@@ -599,7 +599,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 		nextChunk := NewHistogramChunk()
 		app, err := nextChunk.Appender()
 		require.NoError(t, err)
-		newChunk, recoded, newApp, err := app.AppendHistogram(hApp, ts+1, h2, false)
+		newChunk, recoded, newApp, err := app.AppendHistogram(hApp, 0, ts+1, h2, false)
 		require.NoError(t, err)
 		require.Nil(t, newChunk)
 		require.False(t, recoded)
@@ -629,7 +629,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 		nextChunk := NewHistogramChunk()
 		app, err := nextChunk.Appender()
 		require.NoError(t, err)
-		newChunk, recoded, newApp, err := app.AppendHistogram(hApp, ts+1, h2, false)
+		newChunk, recoded, newApp, err := app.AppendHistogram(hApp, 0, ts+1, h2, false)
 		require.NoError(t, err)
 		require.Nil(t, newChunk)
 		require.False(t, recoded)
@@ -776,7 +776,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
 
 func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader, expectHint histogram.CounterResetHint) {
 	oldChunkBytes := oldChunk.Bytes()
-	newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false)
+	newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, 0, ts, h, false)
 	require.Equal(t, oldChunkBytes, oldChunk.Bytes()) // Sanity check that previous chunk is untouched.
 	require.NoError(t, err)
 	require.NotNil(t, newChunk)
@@ -791,7 +791,7 @@ func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Histogr
 
 func assertNoNewHistogramChunkOnAppend(t *testing.T, currChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
 	prevChunkBytes := currChunk.Bytes()
-	newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false)
+	newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, 0, ts, h, false)
 	require.Greater(t, len(currChunk.Bytes()), len(prevChunkBytes)) // Check that current chunk is bigger than previously.
 	require.NoError(t, err)
 	require.Nil(t, newChunk)
@@ -804,7 +804,7 @@ func assertNoNewHistogramChunkOnAppend(t *testing.T, currChunk Chunk, hApp *Hist
 
 func assertRecodedHistogramChunkOnAppend(t *testing.T, prevChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
 	prevChunkBytes := prevChunk.Bytes()
-	newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false)
+	newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, 0, ts, h, false)
 	require.Equal(t, prevChunkBytes, prevChunk.Bytes()) // Sanity check that previous chunk is untouched. This may change in the future if we implement in-place recoding.
 	require.NoError(t, err)
 	require.NotNil(t, newChunk)
@@ -1029,7 +1029,7 @@ func TestHistogramChunkAppendableWithEmptySpan(t *testing.T) {
 			require.NoError(t, err)
 			require.Equal(t, 0, c.NumSamples())
 
-			_, _, _, err = app.AppendHistogram(nil, 1, tc.h1, true)
+			_, _, _, err = app.AppendHistogram(nil, 1, 0, tc.h1, true)
 			require.NoError(t, err)
 			require.Equal(t, 1, c.NumSamples())
 			hApp, _ := app.(*HistogramAppender)
@@ -1172,7 +1172,7 @@ func TestAtFloatHistogram(t *testing.T) {
 	app, err := chk.Appender()
 	require.NoError(t, err)
 	for i := range input {
-		newc, _, _, err := app.AppendHistogram(nil, int64(i), &input[i], false)
+		newc, _, _, err := app.AppendHistogram(nil, 0, int64(i), &input[i], false)
 		require.NoError(t, err)
 		require.Nil(t, newc)
 	}
@@ -1230,7 +1230,7 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
 
 		ts := int64(1234567890)
 
-		chk, _, app, err := app.AppendHistogram(nil, ts, h.Copy(), false)
+		chk, _, app, err := app.AppendHistogram(nil, 0, ts, h.Copy(), false)
 		require.NoError(t, err)
 		require.Nil(t, chk)
 		require.Equal(t, 1, c.NumSamples())
@@ -1471,7 +1471,7 @@ func TestHistogramAppendOnlyErrors(t *testing.T) {
 
 		h := tsdbutil.GenerateTestHistogram(0)
 		var isRecoded bool
-		c, isRecoded, app, err = app.AppendHistogram(nil, 1, h, true)
+		c, isRecoded, app, err = app.AppendHistogram(nil, 0, 1, h, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.NoError(t, err)
@@ -1479,7 +1479,7 @@ func TestHistogramAppendOnlyErrors(t *testing.T) {
 		// Add erroring histogram.
 		h2 := h.Copy()
 		h2.Schema++
-		c, isRecoded, _, err = app.AppendHistogram(nil, 2, h2, true)
+		c, isRecoded, _, err = app.AppendHistogram(nil, 0, 2, h2, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.EqualError(t, err, "histogram schema change")
@@ -1493,7 +1493,7 @@ func TestHistogramAppendOnlyErrors(t *testing.T) {
 
 		h := tsdbutil.GenerateTestHistogram(0)
 		var isRecoded bool
-		c, isRecoded, app, err = app.AppendHistogram(nil, 1, h, true)
+		c, isRecoded, app, err = app.AppendHistogram(nil, 0, 1, h, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.NoError(t, err)
@@ -1501,7 +1501,7 @@ func TestHistogramAppendOnlyErrors(t *testing.T) {
 		// Add erroring histogram.
 		h2 := h.Copy()
 		h2.CounterResetHint = histogram.CounterReset
-		c, isRecoded, _, err = app.AppendHistogram(nil, 2, h2, true)
+		c, isRecoded, _, err = app.AppendHistogram(nil, 0, 2, h2, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.EqualError(t, err, "histogram counter reset")
@@ -1515,7 +1515,7 @@ func TestHistogramAppendOnlyErrors(t *testing.T) {
 
 		h := tsdbutil.GenerateTestCustomBucketsHistogram(0)
 		var isRecoded bool
-		c, isRecoded, app, err = app.AppendHistogram(nil, 1, h, true)
+		c, isRecoded, app, err = app.AppendHistogram(nil, 0, 1, h, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.NoError(t, err)
@@ -1523,7 +1523,7 @@ func TestHistogramAppendOnlyErrors(t *testing.T) {
 		// Add erroring histogram.
 		h2 := h.Copy()
 		h2.CustomValues = []float64{0, 1, 2, 3, 4, 5, 6, 7}
-		c, isRecoded, _, err = app.AppendHistogram(nil, 2, h2, true)
+		c, isRecoded, _, err = app.AppendHistogram(nil, 0, 2, h2, true)
 		require.Nil(t, c)
 		require.False(t, isRecoded)
 		require.EqualError(t, err, "histogram counter reset")
@@ -1556,10 +1556,10 @@ func TestHistogramUniqueSpansAfterNextWithAtHistogram(t *testing.T) {
 	app, err := c.Appender()
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendHistogram(nil, 0, h1, false)
+	_, _, _, err = app.AppendHistogram(nil, 0, 0, h1, false)
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendHistogram(nil, 1, h2, false)
+	_, _, _, err = app.AppendHistogram(nil, 0, 1, h2, false)
 	require.NoError(t, err)
 
 	// Create an iterator and advance to the first histogram.
@@ -1607,10 +1607,10 @@ func TestHistogramUniqueSpansAfterNextWithAtFloatHistogram(t *testing.T) {
 	app, err := c.Appender()
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendHistogram(nil, 0, h1, false)
+	_, _, _, err = app.AppendHistogram(nil, 0, 0, h1, false)
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendHistogram(nil, 1, h2, false)
+	_, _, _, err = app.AppendHistogram(nil, 0, 1, h2, false)
 	require.NoError(t, err)
 
 	// Create an iterator and advance to the first histogram.
@@ -1653,10 +1653,10 @@ func TestHistogramCustomValuesInternedAfterNextWithAtHistogram(t *testing.T) {
 	app, err := c.Appender()
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendHistogram(nil, 0, h1, false)
+	_, _, _, err = app.AppendHistogram(nil, 0, 0, h1, false)
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendHistogram(nil, 1, h2, false)
+	_, _, _, err = app.AppendHistogram(nil, 0, 1, h2, false)
 	require.NoError(t, err)
 
 	// Create an iterator and advance to the first histogram.
@@ -1699,10 +1699,10 @@ func TestHistogramCustomValuesInternedAfterNextWithAtFloatHistogram(t *testing.T
 	app, err := c.Appender()
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendHistogram(nil, 0, h1, false)
+	_, _, _, err = app.AppendHistogram(nil, 0, 0, h1, false)
 	require.NoError(t, err)
 
-	_, _, _, err = app.AppendHistogram(nil, 1, h2, false)
+	_, _, _, err = app.AppendHistogram(nil, 0, 1, h2, false)
 	require.NoError(t, err)
 
 	// Create an iterator and advance to the first histogram.
@@ -1754,7 +1754,7 @@ func BenchmarkAppendable(b *testing.B) {
 		b.Fatal(err)
 	}
 
-	_, _, _, err = app.AppendHistogram(nil, 1, h, true)
+	_, _, _, err = app.AppendHistogram(nil, 0, 1, h, true)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -1791,7 +1791,7 @@ func TestIntHistogramEmptyBucketsWithGaps(t *testing.T) {
 	c := NewHistogramChunk()
 	app, err := c.Appender()
 	require.NoError(t, err)
-	_, _, _, err = app.AppendHistogram(nil, 1, h1, false)
+	_, _, _, err = app.AppendHistogram(nil, 0, 1, h1, false)
 	require.NoError(t, err)
 
 	h2 := &histogram.Histogram{
@@ -1804,7 +1804,7 @@ func TestIntHistogramEmptyBucketsWithGaps(t *testing.T) {
 	}
 	require.NoError(t, h2.Validate())
 
-	newC, recoded, _, err := app.AppendHistogram(nil, 2, h2, false)
+	newC, recoded, _, err := app.AppendHistogram(nil, 0, 2, h2, false)
 	require.NoError(t, err)
 	require.True(t, recoded)
 	require.NotNil(t, newC)
@@ -1839,7 +1839,7 @@ func TestHistogramIteratorFailIfSchemaInValid(t *testing.T) {
 			app, err := c.Appender()
 			require.NoError(t, err)
 
-			_, _, _, err = app.AppendHistogram(nil, 1, h, false)
+			_, _, _, err = app.AppendHistogram(nil, 0, 1, h, false)
 			require.NoError(t, err)
 
 			it := c.Iterator(nil)
@@ -1868,7 +1868,7 @@ func TestHistogramIteratorReduceSchema(t *testing.T) {
 			app, err := c.Appender()
 			require.NoError(t, err)
 
-			_, _, _, err = app.AppendHistogram(nil, 1, h, false)
+			_, _, _, err = app.AppendHistogram(nil, 0, 1, h, false)
 			require.NoError(t, err)
 
 			it := c.Iterator(nil)

tsdb/chunkenc/xor.go

@@ -158,7 +158,7 @@ type xorAppender struct {
 	trailing uint8
 }
 
-func (a *xorAppender) Append(t int64, v float64) {
+func (a *xorAppender) Append(_, t int64, v float64) {
 	var tDelta uint64
 	num := binary.BigEndian.Uint16(a.b.bytes())
 	switch num {
@@ -225,11 +225,11 @@ func (a *xorAppender) writeVDelta(v float64) {
 	xorWrite(a.b, v, a.v, &a.leading, &a.trailing)
 }
 
-func (*xorAppender) AppendHistogram(*HistogramAppender, int64, *histogram.Histogram, bool) (Chunk, bool, Appender, error) {
+func (*xorAppender) AppendHistogram(*HistogramAppender, int64, int64, *histogram.Histogram, bool) (Chunk, bool, Appender, error) {
 	panic("appended a histogram sample to a float chunk")
 }
 
-func (*xorAppender) AppendFloatHistogram(*FloatHistogramAppender, int64, *histogram.FloatHistogram, bool) (Chunk, bool, Appender, error) {
+func (*xorAppender) AppendFloatHistogram(*FloatHistogramAppender, int64, int64, *histogram.FloatHistogram, bool) (Chunk, bool, Appender, error) {
 	panic("appended a float histogram sample to a float chunk")
 }
 
@@ -277,6 +277,10 @@ func (it *xorIterator) AtT() int64 {
 	return it.t
 }
 
+func (*xorIterator) AtST() int64 {
+	return 0
+}
+
 func (it *xorIterator) Err() error {
 	return it.err
 }

tsdb/chunkenc/xor_test.go

@@ -24,7 +24,7 @@ func BenchmarkXorRead(b *testing.B) {
 	app, err := c.Appender()
 	require.NoError(b, err)
 	for i := int64(0); i < 120*1000; i += 1000 {
-		app.Append(i, float64(i)+float64(i)/10+float64(i)/100+float64(i)/1000)
+		app.Append(0, i, float64(i)+float64(i)/10+float64(i)/100+float64(i)/1000)
 	}
 
 	b.ReportAllocs()

tsdb/chunks/chunks.go

@@ -25,7 +25,6 @@ import (
 	"strconv"
 
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
-	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 	"github.com/prometheus/prometheus/tsdb/fileutil"
 )
 
@@ -136,6 +135,7 @@ type Meta struct {
 }
 
 // ChunkFromSamples requires all samples to have the same type.
+// TODO(krajorama): test with ST when chunk formats support it.
 func ChunkFromSamples(s []Sample) (Meta, error) {
 	return ChunkFromSamplesGeneric(SampleSlice(s))
 }
@@ -165,9 +165,9 @@ func ChunkFromSamplesGeneric(s Samples) (Meta, error) {
 	for i := 0; i < s.Len(); i++ {
 		switch sampleType {
 		case chunkenc.ValFloat:
-			ca.Append(s.Get(i).T(), s.Get(i).F())
+			ca.Append(s.Get(i).ST(), s.Get(i).T(), s.Get(i).F())
 		case chunkenc.ValHistogram:
-			newChunk, _, ca, err = ca.AppendHistogram(nil, s.Get(i).T(), s.Get(i).H(), false)
+			newChunk, _, ca, err = ca.AppendHistogram(nil, s.Get(i).ST(), s.Get(i).T(), s.Get(i).H(), false)
 			if err != nil {
 				return emptyChunk, err
 			}
@@ -175,7 +175,7 @@ func ChunkFromSamplesGeneric(s Samples) (Meta, error) {
 				return emptyChunk, errors.New("did not expect to start a second chunk")
 			}
 		case chunkenc.ValFloatHistogram:
-			newChunk, _, ca, err = ca.AppendFloatHistogram(nil, s.Get(i).T(), s.Get(i).FH(), false)
+			newChunk, _, ca, err = ca.AppendFloatHistogram(nil, s.Get(i).ST(), s.Get(i).T(), s.Get(i).FH(), false)
 			if err != nil {
 				return emptyChunk, err
 			}
@@ -431,13 +431,15 @@ func cutSegmentFile(dirFile *os.File, magicNumber uint32, chunksFormat byte, all
 	}
 	defer func() {
 		if returnErr != nil {
-			errs := tsdb_errors.NewMulti(returnErr)
+			errs := []error{
+				returnErr,
+			}
 			if f != nil {
-				errs.Add(f.Close())
+				errs = append(errs, f.Close())
 			}
 			// Calling RemoveAll on a non-existent file does not return error.
-			errs.Add(os.RemoveAll(ptmp))
-			returnErr = errs.Err()
+			errs = append(errs, os.RemoveAll(ptmp))
+			returnErr = errors.Join(errs...)
 		}
 	}()
 	if allocSize > 0 {
@@ -665,10 +667,10 @@ func NewDirReader(dir string, pool chunkenc.Pool) (*Reader, error) {
 	for _, fn := range files {
 		f, err := fileutil.OpenMmapFile(fn)
 		if err != nil {
-			return nil, tsdb_errors.NewMulti(
+			return nil, errors.Join(
 				fmt.Errorf("mmap files: %w", err),
-				tsdb_errors.CloseAll(cs),
-			).Err()
+				closeAll(cs),
+			)
 		}
 		cs = append(cs, f)
 		bs = append(bs, realByteSlice(f.Bytes()))
@@ -676,16 +678,16 @@ func NewDirReader(dir string, pool chunkenc.Pool) (*Reader, error) {
 
 	reader, err := newReader(bs, cs, pool)
 	if err != nil {
-		return nil, tsdb_errors.NewMulti(
+		return nil, errors.Join(
 			err,
-			tsdb_errors.CloseAll(cs),
-		).Err()
+			closeAll(cs),
+		)
 	}
 	return reader, nil
 }
 
 func (s *Reader) Close() error {
-	return tsdb_errors.CloseAll(s.cs)
+	return closeAll(s.cs)
 }
 
 // Size returns the size of the chunks.
@@ -774,3 +776,12 @@ func sequenceFiles(dir string) ([]string, error) {
 	}
 	return res, nil
 }
+
+// closeAll closes all given closers while recording error in MultiError.
+func closeAll(cs []io.Closer) error {
+	var errs []error
+	for _, c := range cs {
+		errs = append(errs, c.Close())
+	}
+	return errors.Join(errs...)
+}

tsdb/chunks/head_chunks.go

@@ -32,7 +32,6 @@ import (
 	"go.uber.org/atomic"
 
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
-	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 	"github.com/prometheus/prometheus/tsdb/fileutil"
 )
 
@@ -304,7 +303,7 @@ func (cdm *ChunkDiskMapper) openMMapFiles() (returnErr error) {
 	cdm.closers = map[int]io.Closer{}
 	defer func() {
 		if returnErr != nil {
-			returnErr = tsdb_errors.NewMulti(returnErr, closeAllFromMap(cdm.closers)).Err()
+			returnErr = errors.Join(returnErr, closeAllFromMap(cdm.closers))
 
 			cdm.mmappedChunkFiles = nil
 			cdm.closers = nil
@@ -614,7 +613,7 @@ func (cdm *ChunkDiskMapper) cut() (seq, offset int, returnErr error) {
 		// The file should not be closed if there is no error,
 		// its kept open in the ChunkDiskMapper.
 		if returnErr != nil {
-			returnErr = tsdb_errors.NewMulti(returnErr, newFile.Close()).Err()
+			returnErr = errors.Join(returnErr, newFile.Close())
 		}
 	}()
 
@@ -970,7 +969,7 @@ func (cdm *ChunkDiskMapper) Truncate(fileNo uint32) error {
 	}
 	cdm.readPathMtx.RUnlock()
 
-	errs := tsdb_errors.NewMulti()
+	var errs []error
 	// Cut a new file only if the current file has some chunks.
 	if cdm.curFileSize() > HeadChunkFileHeaderSize {
 		// There is a known race condition here because between the check of curFileSize() and the call to CutNewFile()
@@ -979,7 +978,7 @@ func (cdm *ChunkDiskMapper) Truncate(fileNo uint32) error {
 		cdm.CutNewFile()
 	}
 	pendingDeletes, err := cdm.deleteFiles(removedFiles)
-	errs.Add(err)
+	errs = append(errs, err)
 
 	if len(chkFileIndices) == len(removedFiles) {
 		// All files were deleted. Reset the current sequence.
@@ -1003,7 +1002,7 @@ func (cdm *ChunkDiskMapper) Truncate(fileNo uint32) error {
 		cdm.evtlPosMtx.Unlock()
 	}
 
-	return errs.Err()
+	return errors.Join(errs...)
 }
 
 // deleteFiles deletes the given file sequences in order of the sequence.
@@ -1098,23 +1097,23 @@ func (cdm *ChunkDiskMapper) Close() error {
 	}
 	cdm.closed = true
 
-	errs := tsdb_errors.NewMulti(
+	errs := []error{
 		closeAllFromMap(cdm.closers),
 		cdm.finalizeCurFile(),
 		cdm.dir.Close(),
-	)
+	}
 	cdm.mmappedChunkFiles = map[int]*mmappedChunkFile{}
 	cdm.closers = map[int]io.Closer{}
 
-	return errs.Err()
+	return errors.Join(errs...)
 }
 
 func closeAllFromMap(cs map[int]io.Closer) error {
-	errs := tsdb_errors.NewMulti()
+	var errs []error
 	for _, c := range cs {
-		errs.Add(c.Close())
+		errs = append(errs, c.Close())
 	}
-	return errs.Err()
+	return errors.Join(errs...)
 }
 
 const inBufferShards = 128 // 128 is a randomly chosen number.