diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d25176252..d1f3a0c988 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3 + - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4 - uses: ./.github/promci/actions/setup_environment with: enable_npm: true @@ -37,7 +37,7 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3 + - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4 - uses: ./.github/promci/actions/setup_environment - run: go test --tags=dedupelabels ./... - run: go test --tags=slicelabels -race ./cmd/prometheus ./model/textparse ./prompb/... @@ -81,7 +81,7 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3 + - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4 - uses: ./.github/promci/actions/setup_environment with: enable_go: false @@ -146,7 +146,7 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3 + - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4 - uses: ./.github/promci/actions/build with: promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386" @@ -173,7 +173,7 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3 + - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4 - uses: ./.github/promci/actions/build with: parallelism: 12 @@ -212,7 +212,7 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3 + - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4 - uses: ./.github/promci/actions/setup_environment with: enable_npm: true @@ -270,7 +270,7 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3 + - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4 - uses: ./.github/promci/actions/publish_main with: docker_hub_login: ${{ secrets.docker_hub_login }} @@ -289,7 +289,7 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3 + - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4 - uses: ./.github/promci/actions/publish_release with: docker_hub_login: ${{ secrets.docker_hub_login }} @@ -306,7 +306,7 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: prometheus/promci@c0916f0a41f13444612a8f0f5e700ea34edd7c19 # v0.5.3 + - uses: prometheus/promci@fc721ff8497a70a93a881cd552b71af7fb3a9d53 # v0.5.4 - name: Install nodejs uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 with: diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index f9f7abafd6..55ab70dbac 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -1,30 +1,47 @@ -name: CIFuzz +name: fuzzing on: workflow_call: permissions: contents: read jobs: - Fuzzing: + fuzzing: + name: Run Go Fuzz Tests runs-on: ubuntu-latest + strategy: + matrix: + fuzz_test: [FuzzParseMetricText, FuzzParseOpenMetric, FuzzParseMetricSelector, FuzzParseExpr] steps: - - name: Build Fuzzers - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@4bf20ff8dfda18ad651583ebca9fb17a7ce1940a # master + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - oss-fuzz-project-name: "prometheus" - dry-run: false - - name: Run Fuzzers - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@4bf20ff8dfda18ad651583ebca9fb17a7ce1940a # master - # Note: Regularly check for updates to the pinned commit hash at: - # https://github.com/google/oss-fuzz/tree/master/infra/cifuzz/actions/run_fuzzers + persist-credentials: false + - name: Install Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 with: - oss-fuzz-project-name: "prometheus" - fuzz-seconds: 600 - dry-run: false - - name: Upload Crash - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - if: failure() && steps.build.outcome == 'success' + go-version: 1.25.x + - name: Run Fuzzing + run: go test -fuzz=${{ matrix.fuzz_test }}$ -fuzztime=5m ./util/fuzzing + continue-on-error: true + id: fuzz + - name: Upload Crash Artifacts + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + if: failure() with: - name: artifacts - path: ./out/artifacts + name: fuzz-artifacts-${{ matrix.fuzz_test }} + path: util/fuzzing/testdata/fuzz/${{ matrix.fuzz_test }} + fuzzing_status: + # This status check aggregates the individual matrix jobs of the fuzzing + # step into a final status. Fails if a single matrix job fails, succeeds if + # all matrix jobs succeed. + name: Fuzzing + runs-on: ubuntu-latest + needs: [fuzzing] + if: always() + steps: + - name: Successful fuzzing + if: ${{ !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled')) }} + run: exit 0 + - name: Failing or cancelled fuzzing + if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} + run: exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d43bb24720..a1afb0af59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,7 +54,7 @@ * [FEATURE] OAuth2: support jwt-bearer grant-type (RFC7523 3.1). #17592 * [FEATURE] Dockerfile: Add OpenContainers spec labels to Dockerfile. #16483 * [FEATURE] SD: Add unified AWS service discovery for ec2, lightsail and ecs services. #17406 -* [FEATURE] Native histograms are now a stable, but optional feature, use the `scrape_native_histogram` config setting. #17232 #17315 +* [FEATURE] Native histograms are now a stable, but optional feature, use the `scrape_native_histograms` config setting. #17232 #17315 * [FEATURE] UI: Support anchored and smoothed keyword in promql editor. #17239 * [FEATURE] UI: Show detailed relabeling steps for each discovered target. #17337 * [FEATURE] Alerting: Add urlQueryEscape to template functions. #17403 diff --git a/Dockerfile b/Dockerfile index 071e7441e3..98712d8f9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ LABEL org.opencontainers.image.authors="The Prometheus Authors" \ org.opencontainers.image.source="https://github.com/prometheus/prometheus" \ org.opencontainers.image.url="https://github.com/prometheus/prometheus" \ org.opencontainers.image.documentation="https://prometheus.io/docs" \ - org.opencontainers.image.licenses="Apache License 2.0" + org.opencontainers.image.licenses="Apache License 2.0" \ + io.prometheus.image.variant="busybox" ARG ARCH="amd64" ARG OS="linux" diff --git a/Dockerfile.distroless b/Dockerfile.distroless new file mode 100644 index 0000000000..0ee184a91c --- /dev/null +++ b/Dockerfile.distroless @@ -0,0 +1,29 @@ +ARG DISTROLESS_ARCH="amd64" + +# Use DISTROLESS_ARCH for base image selection (handles armv7->arm mapping). +FROM gcr.io/distroless/static-debian13:nonroot-${DISTROLESS_ARCH} +# Base image sets USER to 65532:65532 (nonroot user). + +ARG ARCH="amd64" +ARG OS="linux" + +LABEL org.opencontainers.image.authors="The Prometheus Authors" +LABEL org.opencontainers.image.vendor="Prometheus" +LABEL org.opencontainers.image.title="Prometheus" +LABEL org.opencontainers.image.description="The Prometheus monitoring system and time series database" +LABEL org.opencontainers.image.source="https://github.com/prometheus/prometheus" +LABEL org.opencontainers.image.url="https://github.com/prometheus/prometheus" +LABEL org.opencontainers.image.documentation="https://prometheus.io/docs" +LABEL org.opencontainers.image.licenses="Apache License 2.0" +LABEL io.prometheus.image.variant="distroless" + +COPY documentation/examples/prometheus.yml /etc/prometheus/prometheus.yml +COPY LICENSE NOTICE npm_licenses.tar.bz2 / +COPY .build/${OS}-${ARCH}/prometheus /bin/prometheus +COPY .build/${OS}-${ARCH}/promtool /bin/promtool + +WORKDIR /prometheus +EXPOSE 9090 +ENTRYPOINT [ "/bin/prometheus" ] +CMD [ "--config.file=/etc/prometheus/prometheus.yml", \ + "--storage.tsdb.path=/prometheus" ] diff --git a/Makefile b/Makefile index 8c15ceb2e9..8bc4a3dcaa 100644 --- a/Makefile +++ b/Makefile @@ -220,3 +220,8 @@ check-node-version: bump-go-version: @echo ">> bumping Go minor version" @./scripts/bump_go_version.sh + +.PHONY: generate-fuzzing-seed-corpus +generate-fuzzing-seed-corpus: + @echo ">> Generating fuzzing seed corpus" + @$(GO) generate -tags fuzzing ./util/fuzzing/corpus_gen diff --git a/Makefile.common b/Makefile.common index 7beae6e58f..b8c9b3844c 100644 --- a/Makefile.common +++ b/Makefile.common @@ -82,11 +82,32 @@ endif PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) -DOCKERFILE_PATH ?= ./Dockerfile DOCKERBUILD_CONTEXT ?= ./ DOCKER_REPO ?= prom +# Check if deprecated DOCKERFILE_PATH is set +ifdef DOCKERFILE_PATH +$(error DOCKERFILE_PATH is deprecated. Use DOCKERFILE_VARIANTS ?= $(DOCKERFILE_PATH) in the Makefile) +endif + DOCKER_ARCHS ?= amd64 +DOCKERFILE_VARIANTS ?= Dockerfile $(wildcard Dockerfile.*) + +# Function to extract variant from Dockerfile label. +# Returns the variant name from io.prometheus.image.variant label, or "default" if not found. +define dockerfile_variant +$(strip $(or $(shell sed -n 's/.*io\.prometheus\.image\.variant="\([^"]*\)".*/\1/p' $(1)),default)) +endef + +# Check for duplicate variant names (including default for Dockerfiles without labels). +DOCKERFILE_VARIANT_NAMES := $(foreach df,$(DOCKERFILE_VARIANTS),$(call dockerfile_variant,$(df))) +DOCKERFILE_VARIANT_NAMES_SORTED := $(sort $(DOCKERFILE_VARIANT_NAMES)) +ifneq ($(words $(DOCKERFILE_VARIANT_NAMES)),$(words $(DOCKERFILE_VARIANT_NAMES_SORTED))) +$(error Duplicate variant names found. Each Dockerfile must have a unique io.prometheus.image.variant label, and only one can be without a label (default)) +endif + +# Build variant:dockerfile pairs for shell iteration. +DOCKERFILE_VARIANTS_WITH_NAMES := $(foreach df,$(DOCKERFILE_VARIANTS),$(call dockerfile_variant,$(df)):$(df)) BUILD_DOCKER_ARCHS = $(addprefix common-docker-,$(DOCKER_ARCHS)) PUBLISH_DOCKER_ARCHS = $(addprefix common-docker-publish-,$(DOCKER_ARCHS)) @@ -226,28 +247,110 @@ common-docker-repo-name: .PHONY: common-docker $(BUILD_DOCKER_ARCHS) common-docker: $(BUILD_DOCKER_ARCHS) $(BUILD_DOCKER_ARCHS): common-docker-%: - docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \ - -f $(DOCKERFILE_PATH) \ - --build-arg ARCH="$*" \ - --build-arg OS="linux" \ - $(DOCKERBUILD_CONTEXT) + @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ + dockerfile=$${variant#*:}; \ + variant_name=$${variant%%:*}; \ + distroless_arch="$*"; \ + if [ "$*" = "armv7" ]; then \ + distroless_arch="arm"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Building default variant ($$variant_name) for linux-$* using $$dockerfile"; \ + docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \ + -f $$dockerfile \ + --build-arg ARCH="$*" \ + --build-arg OS="linux" \ + --build-arg DISTROLESS_ARCH="$$distroless_arch" \ + $(DOCKERBUILD_CONTEXT); \ + if [ "$$variant_name" != "default" ]; then \ + echo "Tagging default variant with $$variant_name suffix"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \ + "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \ + fi; \ + else \ + echo "Building $$variant_name variant for linux-$* using $$dockerfile"; \ + docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" \ + -f $$dockerfile \ + --build-arg ARCH="$*" \ + --build-arg OS="linux" \ + --build-arg DISTROLESS_ARCH="$$distroless_arch" \ + $(DOCKERBUILD_CONTEXT); \ + fi; \ + done .PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS) common-docker-publish: $(PUBLISH_DOCKER_ARCHS) $(PUBLISH_DOCKER_ARCHS): common-docker-publish-%: - docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" + @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ + dockerfile=$${variant#*:}; \ + variant_name=$${variant%%:*}; \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Pushing $$variant_name variant for linux-$*"; \ + docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Pushing default variant ($$variant_name) for linux-$*"; \ + docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)"; \ + fi; \ + if [ "$(DOCKER_IMAGE_TAG)" = "latest" ]; then \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Pushing $$variant_name variant version tags for linux-$*"; \ + docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Pushing default variant version tag for linux-$*"; \ + docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"; \ + fi; \ + fi; \ + done DOCKER_MAJOR_VERSION_TAG = $(firstword $(subst ., ,$(shell cat VERSION))) .PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS) common-docker-tag-latest: $(TAG_DOCKER_ARCHS) $(TAG_DOCKER_ARCHS): common-docker-tag-latest-%: - docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest" - docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)" + @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ + dockerfile=$${variant#*:}; \ + variant_name=$${variant%%:*}; \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Tagging $$variant_name variant for linux-$* as latest"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest-$$variant_name"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Tagging default variant ($$variant_name) for linux-$* as latest"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest"; \ + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"; \ + fi; \ + done .PHONY: common-docker-manifest common-docker-manifest: - DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG)) - DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" + @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ + dockerfile=$${variant#*:}; \ + variant_name=$${variant%%:*}; \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Creating manifest for $$variant_name variant"; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name); \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Creating default variant ($$variant_name) manifest"; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG)); \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)"; \ + fi; \ + if [ "$(DOCKER_IMAGE_TAG)" = "latest" ]; then \ + if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ + echo "Creating manifest for $$variant_name variant version tag"; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name); \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \ + fi; \ + if [ "$$dockerfile" = "Dockerfile" ]; then \ + echo "Creating default variant version tag manifest"; \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):v$(DOCKER_MAJOR_VERSION_TAG)); \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):v$(DOCKER_MAJOR_VERSION_TAG)"; \ + fi; \ + fi; \ + done .PHONY: promu promu: $(PROMU) diff --git a/RELEASE.md b/RELEASE.md index c7375b35aa..5a8f8601ab 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -7,19 +7,20 @@ This page describes the release process and the currently planned schedule for u Release cadence of first pre-releases being cut is 6 weeks. Please see [the v2.55 RELEASE.md](https://github.com/prometheus/prometheus/blob/release-2.55/RELEASE.md) for the v2 release series schedule. -| release series | date of first pre-release (year-month-day) | release shepherd | -|----------------|--------------------------------------------|------------------------------------| -| v3.0 | 2024-11-14 | Jan Fajerski (GitHub: @jan--f) | -| v3.1 | 2024-12-17 | Bryan Boreham (GitHub: @bboreham) | -| v3.2 | 2025-01-28 | Jan Fajerski (GitHub: @jan--f) | -| v3.3 | 2025-03-11 | Ayoub Mrini (Github: @machine424) | -| v3.4 | 2025-04-29 | Jan-Otto Kröpke (Github: @jkroepke)| -| v3.5 LTS | 2025-06-03 | Bryan Boreham (GitHub: @bboreham) | -| v3.6 | 2025-08-01 | Ayoub Mrini (Github: @machine424) | -| v3.7 | 2025-09-25 | Arthur Sens and George Krajcsovits (Github: @ArthurSens and @krajorama)| -| v3.8 | 2025-11-06 | Jan Fajerski (GitHub: @jan--f) | -| v3.9 | 2025-12-18 | Bryan Boreham (GitHub: @bboreham) | -| v3.10 | 2026-02-05 | **volunteer welcome** | +| release series | date of first pre-release (year-month-day) | release shepherd | +|----------------|--------------------------------------------|-------------------------------------------------------------------------| +| v3.0 | 2024-11-14 | Jan Fajerski (GitHub: @jan--f) | +| v3.1 | 2024-12-17 | Bryan Boreham (GitHub: @bboreham) | +| v3.2 | 2025-01-28 | Jan Fajerski (GitHub: @jan--f) | +| v3.3 | 2025-03-11 | Ayoub Mrini (Github: @machine424) | +| v3.4 | 2025-04-29 | Jan-Otto Kröpke (Github: @jkroepke) | +| v3.5 LTS | 2025-06-03 | Bryan Boreham (GitHub: @bboreham) | +| v3.6 | 2025-08-01 | Ayoub Mrini (Github: @machine424) | +| v3.7 | 2025-09-25 | Arthur Sens and George Krajcsovits (Github: @ArthurSens and @krajorama) | +| v3.8 | 2025-11-06 | Jan Fajerski (GitHub: @jan--f) | +| v3.9 | 2025-12-18 | Bryan Boreham (GitHub: @bboreham) | +| v3.10 | 2026-02-05 | Ganesh Vernekar (Github: @codesome) | +| v3.11 | 2026-03-19 | **volunteer welcome** | If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice. diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 0fa48c72b9..53584085ec 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -281,6 +281,9 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error { case "promql-extended-range-selectors": parser.EnableExtendedRangeSelectors = true logger.Info("Experimental PromQL extended range selectors enabled.") + case "promql-binop-fill-modifiers": + parser.EnableBinopFillModifiers = true + logger.Info("Experimental PromQL binary operator fill modifiers enabled.") case "": continue case "old-ui": @@ -578,7 +581,7 @@ func main() { a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates."). Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval) - a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). + a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). Default("").StringsVar(&cfg.featureList) a.Flag("agent", "Run Prometheus in 'Agent mode'.").BoolVar(&agentMode) @@ -1573,7 +1576,7 @@ func reloadConfig(filename string, enableExemplarStorage bool, logger *slog.Logg logger.Error("Failed to apply configuration", "err", err) failed = true } - timingsLogger = timingsLogger.With((rl.name), time.Since(rstart)) + timingsLogger = timingsLogger.With(rl.name, time.Since(rstart)) } if failed { return fmt.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename) @@ -1747,6 +1750,14 @@ func (s *readyStorage) Appender(ctx context.Context) storage.Appender { return notReadyAppender{} } +// AppenderV2 implements the Storage interface. +func (s *readyStorage) AppenderV2(ctx context.Context) storage.AppenderV2 { + if x := s.get(); x != nil { + return x.AppenderV2(ctx) + } + return notReadyAppenderV2{} +} + type notReadyAppender struct{} // SetOptions does nothing in this appender implementation. @@ -1780,6 +1791,15 @@ func (notReadyAppender) Commit() error { return tsdb.ErrNotReady } func (notReadyAppender) Rollback() error { return tsdb.ErrNotReady } +type notReadyAppenderV2 struct{} + +func (notReadyAppenderV2) Append(storage.SeriesRef, labels.Labels, int64, int64, float64, *histogram.Histogram, *histogram.FloatHistogram, storage.AOptions) (storage.SeriesRef, error) { + return 0, tsdb.ErrNotReady +} +func (notReadyAppenderV2) Commit() error { return tsdb.ErrNotReady } + +func (notReadyAppenderV2) Rollback() error { return tsdb.ErrNotReady } + // Close implements the Storage interface. func (s *readyStorage) Close() error { if x := s.get(); x != nil { diff --git a/cmd/prometheus/testdata/features.json b/cmd/prometheus/testdata/features.json index 145bb04d77..4c893daae2 100644 --- a/cmd/prometheus/testdata/features.json +++ b/cmd/prometheus/testdata/features.json @@ -28,6 +28,9 @@ "by": true, "delayed_name_removal": false, "duration_expr": false, + "fill": false, + "fill_left": false, + "fill_right": false, "group_left": true, "group_right": true, "ignoring": true, diff --git a/cmd/promtool/backfill.go b/cmd/promtool/backfill.go index f04a76b0a5..e7a9a7f18a 100644 --- a/cmd/promtool/backfill.go +++ b/cmd/promtool/backfill.go @@ -27,7 +27,6 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/textparse" "github.com/prometheus/prometheus/tsdb" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" ) func getMinAndMaxTimestamps(p textparse.Parser) (int64, int64, error) { @@ -94,7 +93,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn return err } defer func() { - returnErr = tsdb_errors.NewMulti(returnErr, db.Close()).Err() + returnErr = errors.Join(returnErr, db.Close()) }() var ( @@ -125,7 +124,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn return fmt.Errorf("block writer: %w", err) } defer func() { - err = tsdb_errors.NewMulti(err, w.Close()).Err() + err = errors.Join(err, w.Close()) }() ctx := context.Background() diff --git a/cmd/promtool/rules.go b/cmd/promtool/rules.go index 3960206f6b..bb45178e9c 100644 --- a/cmd/promtool/rules.go +++ b/cmd/promtool/rules.go @@ -15,6 +15,7 @@ package main import ( "context" + "errors" "fmt" "log/slog" "time" @@ -28,7 +29,6 @@ import ( "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" ) const maxSamplesInMemory = 5000 @@ -143,7 +143,7 @@ func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName var closed bool defer func() { if !closed { - err = tsdb_errors.NewMulti(err, w.Close()).Err() + err = errors.Join(err, w.Close()) } }() app := newMultipleAppender(ctx, w) @@ -181,7 +181,7 @@ func (importer *ruleImporter) importRule(ctx context.Context, ruleExpr, ruleName if err := app.flushAndCommit(ctx); err != nil { return fmt.Errorf("flush and commit: %w", err) } - err = tsdb_errors.NewMulti(err, w.Close()).Err() + err = errors.Join(err, w.Close()) closed = true } diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go index 9ccd1da714..d0016ec0aa 100644 --- a/cmd/promtool/tsdb.go +++ b/cmd/promtool/tsdb.go @@ -43,7 +43,6 @@ import ( "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" "github.com/prometheus/prometheus/tsdb/index" ) @@ -339,7 +338,7 @@ func listBlocks(path string, humanReadable bool) error { return err } defer func() { - err = tsdb_errors.NewMulti(err, db.Close()).Err() + err = errors.Join(err, db.Close()) }() blocks, err := db.Blocks() if err != nil { @@ -425,7 +424,7 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten return err } defer func() { - err = tsdb_errors.NewMulti(err, db.Close()).Err() + err = errors.Join(err, db.Close()) }() meta := block.Meta() @@ -625,7 +624,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb. return err } defer func() { - err = tsdb_errors.NewMulti(err, chunkr.Close()).Err() + err = errors.Join(err, chunkr.Close()) }() totalChunks := 0 @@ -713,7 +712,7 @@ func dumpTSDBData(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt return err } defer func() { - err = tsdb_errors.NewMulti(err, db.Close()).Err() + err = errors.Join(err, db.Close()) }() q, err := db.Querier(mint, maxt) if err != nil { @@ -743,7 +742,7 @@ func dumpTSDBData(ctx context.Context, dbDir, sandboxDirRoot string, mint, maxt } if ws := ss.Warnings(); len(ws) > 0 { - return tsdb_errors.NewMulti(ws.AsErrors()...).Err() + return errors.Join(ws.AsErrors()...) } if ss.Err() != nil { diff --git a/discovery/aws/aws.go b/discovery/aws/aws.go index 1ac97b3c9e..be6b4dabbe 100644 --- a/discovery/aws/aws.go +++ b/discovery/aws/aws.go @@ -101,7 +101,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { switch c.Role { case RoleEC2: if c.EC2SDConfig == nil { - c.EC2SDConfig = &DefaultEC2SDConfig + ec2Config := DefaultEC2SDConfig + c.EC2SDConfig = &ec2Config } c.EC2SDConfig.HTTPClientConfig = c.HTTPClientConfig if c.Region != "" { @@ -133,7 +134,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { } case RoleECS: if c.ECSSDConfig == nil { - c.ECSSDConfig = &DefaultECSSDConfig + ecsConfig := DefaultECSSDConfig + c.ECSSDConfig = &ecsConfig } c.ECSSDConfig.HTTPClientConfig = c.HTTPClientConfig if c.Region != "" { @@ -165,7 +167,8 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(any) error) error { } case RoleLightsail: if c.LightsailSDConfig == nil { - c.LightsailSDConfig = &DefaultLightsailSDConfig + lightsailConfig := DefaultLightsailSDConfig + c.LightsailSDConfig = &lightsailConfig } c.LightsailSDConfig.HTTPClientConfig = c.HTTPClientConfig if c.Region != "" { diff --git a/discovery/aws/aws_test.go b/discovery/aws/aws_test.go index a2f03a8b99..dc1f2044ec 100644 --- a/discovery/aws/aws_test.go +++ b/discovery/aws/aws_test.go @@ -20,7 +20,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) func TestRoleUnmarshalYAML(t *testing.T) { @@ -177,3 +177,109 @@ port: 9300`, }) } } + +// TestMultipleSDConfigsDoNotShareState verifies that multiple AWS SD configs +// don't share the same underlying configuration object. This was a bug where +// all configs pointed to the same global default, causing port and other +// settings from one job to overwrite settings in another job. +func TestMultipleSDConfigsDoNotShareState(t *testing.T) { + tests := []struct { + name string + yaml string + validateFunc func(t *testing.T, cfg1, cfg2 *SDConfig) + }{ + { + name: "EC2MultipleJobsDifferentPorts", + yaml: ` +- role: ec2 + region: us-west-2 + port: 9100 + filters: + - name: tag:Name + values: [host-1] +- role: ec2 + region: us-west-2 + port: 9101 + filters: + - name: tag:Name + values: [host-2]`, + validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) { + require.Equal(t, RoleEC2, cfg1.Role) + require.Equal(t, RoleEC2, cfg2.Role) + require.NotNil(t, cfg1.EC2SDConfig) + require.NotNil(t, cfg2.EC2SDConfig) + + // Verify ports are different and not shared + require.Equal(t, 9100, cfg1.EC2SDConfig.Port) + require.Equal(t, 9101, cfg2.EC2SDConfig.Port) + + // Verify filters are different and not shared + require.Len(t, cfg1.EC2SDConfig.Filters, 1) + require.Len(t, cfg2.EC2SDConfig.Filters, 1) + require.Equal(t, []string{"host-1"}, cfg1.EC2SDConfig.Filters[0].Values) + require.Equal(t, []string{"host-2"}, cfg2.EC2SDConfig.Filters[0].Values) + + // Most importantly: verify they're not the same pointer + require.NotSame(t, cfg1.EC2SDConfig, cfg2.EC2SDConfig, + "EC2SDConfig objects should not share the same memory address") + }, + }, + { + name: "ECSMultipleJobsDifferentPorts", + yaml: ` +- role: ecs + region: us-east-1 + port: 8080 + clusters: [cluster-a] +- role: ecs + region: us-east-1 + port: 8081 + clusters: [cluster-b]`, + validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) { + require.Equal(t, RoleECS, cfg1.Role) + require.Equal(t, RoleECS, cfg2.Role) + require.NotNil(t, cfg1.ECSSDConfig) + require.NotNil(t, cfg2.ECSSDConfig) + + require.Equal(t, 8080, cfg1.ECSSDConfig.Port) + require.Equal(t, 8081, cfg2.ECSSDConfig.Port) + require.Equal(t, []string{"cluster-a"}, cfg1.ECSSDConfig.Clusters) + require.Equal(t, []string{"cluster-b"}, cfg2.ECSSDConfig.Clusters) + + require.NotSame(t, cfg1.ECSSDConfig, cfg2.ECSSDConfig, + "ECSSDConfig objects should not share the same memory address") + }, + }, + { + name: "LightsailMultipleJobsDifferentPorts", + yaml: ` +- role: lightsail + region: eu-west-1 + port: 7070 +- role: lightsail + region: eu-west-1 + port: 7071`, + validateFunc: func(t *testing.T, cfg1, cfg2 *SDConfig) { + require.Equal(t, RoleLightsail, cfg1.Role) + require.Equal(t, RoleLightsail, cfg2.Role) + require.NotNil(t, cfg1.LightsailSDConfig) + require.NotNil(t, cfg2.LightsailSDConfig) + + require.Equal(t, 7070, cfg1.LightsailSDConfig.Port) + require.Equal(t, 7071, cfg2.LightsailSDConfig.Port) + + require.NotSame(t, cfg1.LightsailSDConfig, cfg2.LightsailSDConfig, + "LightsailSDConfig objects should not share the same memory address") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var configs []SDConfig + require.NoError(t, yaml.Unmarshal([]byte(tt.yaml), &configs)) + require.Len(t, configs, 2) + tt.validateFunc(t, &configs[0], &configs[1]) + }) + } +} diff --git a/docs/command-line/prometheus.md b/docs/command-line/prometheus.md index d4a8cd4f20..251fdfd6a4 100644 --- a/docs/command-line/prometheus.md +++ b/docs/command-line/prometheus.md @@ -59,7 +59,7 @@ The Prometheus monitoring server | --query.timeout | Maximum time a query may take before being aborted. Use with server mode only. | `2m` | | --query.max-concurrency | Maximum number of queries executed concurrently. Use with server mode only. | `20` | | --query.max-samples | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` | -| --enable-feature ... | Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | +| --enable-feature ... | Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | | --agent | Run Prometheus in 'Agent mode'. | | | --log.level | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` | | --log.format | Output format of log messages. One of: [logfmt, json] | `logfmt` | diff --git a/docs/feature_flags.md b/docs/feature_flags.md index af08eebb45..247941c5ce 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -67,12 +67,12 @@ Currently, Prometheus supports start timestamps on the * `PrometheusProto` * `OpenMetrics1.0.0` - + From the above, Prometheus recommends `PrometheusProto`. This is because OpenMetrics 1.0 Start Timestamp information is shared as a `_created` metric and parsing those are prone to errors and expensive (thus, adding an overhead). You also need to be careful to not pollute your Prometheus with extra `_created` metrics. - -Therefore, when `created-timestamp-zero-ingestion` is enabled Prometheus changes the global `scrape_protocols` default configuration option to + +Therefore, when `created-timestamp-zero-ingestion` is enabled Prometheus changes the global `scrape_protocols` default configuration option to `[ PrometheusProto, OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText0.0.4 ]`, resulting in negotiating the Prometheus Protobuf protocol first (unless the `scrape_protocols` option is set to a different value explicitly). Besides enabling this feature in Prometheus, start timestamps need to be exposed by the application being scraped. @@ -288,8 +288,8 @@ when wrong types are used on wrong functions, automatic renames, delta types and ### Behavior with metadata records -When this feature is enabled and the metadata WAL records exists, in an unlikely situation when type or unit are different across those, -the Prometheus outputs intends to prefer the `__type__` and `__unit__` labels values. For example on Remote Write 2.0, +When this feature is enabled and the metadata WAL records exists, in an unlikely situation when type or unit are different across those, +the Prometheus outputs intends to prefer the `__type__` and `__unit__` labels values. For example on Remote Write 2.0, if the metadata record somehow (e.g. due to bug) says "counter", but `__type__="gauge"` the remote time series will be set to a gauge. ## Use Uncached IO @@ -338,9 +338,25 @@ Example query: > **Note for alerting and recording rules:** > The `smoothed` modifier requires samples after the evaluation interval, so using it directly in alerting or recording rules will typically *under-estimate* the result, as future samples are not available at evaluation time. -> To use `smoothed` safely in rules, you **must** apply a `query_offset` to the rule group (see [documentation](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#rule_group)) to ensure the calculation window is fully in the past and all needed samples are available. +> To use `smoothed` safely in rules, you **must** apply a `query_offset` to the rule group (see [documentation](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#rule_group)) to ensure the calculation window is fully in the past and all needed samples are available. > For critical alerting, set the offset to at least one scrape interval; for less critical or more resilient use cases, consider a larger offset (multiple scrape intervals) to tolerate missed scrapes. For more details, see the [design doc](https://github.com/prometheus/proposals/blob/main/proposals/2025-04-04_extended-range-selectors-semantics.md). **Note**: Extended Range Selectors are not supported for subqueries. + +## Binary operator fill modifiers + +`--enable-feature=promql-binop-fill-modifiers` + +Enables experimental `fill()`, `fill_left()`, and `fill_right()` modifiers for PromQL binary operators. These modifiers allow filling in missing matches on either side of a binary operation with a provided default sample value. + +Example query: + +``` + rate(successful_requests[5m]) ++ fill(0) + rate(failed_requests[5m]) +``` + +See [the fill modifiers documentation](querying/operators.md#filling-in-missing-matches) for more details and examples. diff --git a/docs/querying/operators.md b/docs/querying/operators.md index b320d8e86e..b15c02aedc 100644 --- a/docs/querying/operators.md +++ b/docs/querying/operators.md @@ -47,9 +47,9 @@ special values like `NaN`, `+Inf`, and `-Inf`. scalar that is the result of the operator applied to both scalar operands. **Between an instant vector and a scalar**, the operator is applied to the -value of every data sample in the vector. +value of every data sample in the vector. -If the data sample is a float, the operation is performed between that float and the scalar. +If the data sample is a float, the operation is performed between that float and the scalar. For example, if an instant vector of float samples is multiplied by 2, the result is another vector of float samples in which every sample value of the original vector is multiplied by 2. @@ -81,8 +81,9 @@ following: **Between two instant vectors**, a binary arithmetic operator is applied to each entry in the LHS vector and its [matching element](#vector-matching) in the RHS vector. The result is propagated into the result vector with the -grouping labels becoming the output label set. Entries for which no matching -entry in the right-hand vector can be found are not part of the result. +grouping labels becoming the output label set. By default, series for which +no matching entry in the opposite vector can be found are not part of the +result. This behavior can be adjusted using [fill modifiers](#filling-in-missing-matches). If two float samples are matched, the arithmetic operator is applied to the two input values. @@ -97,7 +98,7 @@ If two histogram samples are matched, only `+` and `-` are valid operations, each adding or subtracting all matching bucket populations and the count and the sum of observations. All other operations result in the removal of the corresponding element from the output vector, flagged by an info-level -annotation. The `+` and -` operations should generally only be applied to gauge +annotation. The `+` and `-` operations should generally only be applied to gauge histograms, but PromQL allows them for counter histograms, too, to cover specific use cases, for which special attention is required to avoid problems with unaligned counter resets. (Certain incompatibilities of counter resets can @@ -106,7 +107,7 @@ two counter histograms results in a counter histogram. All other combination of operands and all subtractions result in a gauge histogram. **In any arithmetic binary operation involving vectors**, the metric name is -dropped. This occurs even if `__name__` is explicitly mentioned in `on` +dropped. This occurs even if `__name__` is explicitly mentioned in `on` (see https://github.com/prometheus/prometheus/issues/16631 for further discussion). **For any arithmetic binary operation that may result in a negative @@ -156,9 +157,9 @@ info-level annotation. applied to matching entries. Vector elements for which the expression is not true or which do not find a match on the other side of the expression get dropped from the result, while the others are propagated into a result vector -with the grouping labels becoming the output label set. +with the grouping labels becoming the output label set. -Matches between two float samples work as usual. +Matches between two float samples work as usual. Matches between a float sample and a histogram sample are invalid, and the corresponding element is removed from the result vector, flagged by an info-level @@ -171,8 +172,8 @@ comparison binary operations are again invalid. modifier changes the behavior in the following ways: * Vector elements which find a match on the other side of the expression but for - which the expression is false instead have the value `0` and vector elements - that do find a match and for which the expression is true have the value `1`. + which the expression is false instead have the value `0`, and vector elements + that do find a match and for which the expression is true have the value `1`. (Note that elements with no match or invalid operations involving histogram samples still return no result rather than the value `0`.) * The metric name is dropped. @@ -216,11 +217,10 @@ matching behavior: One-to-one and many-to-one/one-to-many. ### Vector matching keywords -These vector matching keywords allow for matching between series with different label sets -providing: +These vector matching keywords allow for matching between series with different label sets: -* `on` -* `ignoring` +* `on(