From cd98ded6ea2b821c8b14c53372e73a8f3f42b1d2 Mon Sep 17 00:00:00 2001 From: bwplotka Date: Fri, 12 Dec 2025 22:43:51 +0000 Subject: [PATCH 1/5] test: add regression test against remote write handler bad response stats Signed-off-by: bwplotka --- storage/remote/client.go | 3 ++ storage/remote/write_handler.go | 4 ++ storage/remote/write_handler_test.go | 72 ++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/storage/remote/client.go b/storage/remote/client.go index c535ea3425..0f2b5ddca6 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -301,6 +301,9 @@ func (c *Client) Store(ctx context.Context, req []byte, attempt int) (WriteRespo _ = httpResp.Body.Close() }() + // NOTE(bwplotka): Only PRW2 spec defines response HTTP headers. However, spec does not block + // PRW1 from sending them too for reliability. Support this case. + // // TODO(bwplotka): Pass logger and emit debug on error? // Parsing error means there were some response header values we can't parse, // we can continue handling. diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index f8296b4a80..2bc65e8286 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -96,6 +96,10 @@ func isHistogramValidationError(err error) bool { } // Store implements remoteapi.writeStorage interface. +// TODO(bwplotka): Improve remoteapi.Store API. Right now it's confusing if PRWv1 flows should use WriteResponse or not. +// If it's not filled, it will be "confirmed zero" which caused partial error reporting on client side in the past. +// Temporary fix was done to only care about WriteResponse stats for PRW2 (see https://github.com/prometheus/client_golang/pull/1927 +// but better approach would be to only confirm if explicit stats were injected. func (h *writeHandler) Store(r *http.Request, msgType remoteapi.WriteMessageType) (*remoteapi.WriteResponse, error) { // Store receives request with decompressed content in body. body, err := io.ReadAll(r.Body) diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index f1c064c64d..2610142db9 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -1510,3 +1510,75 @@ func TestHistogramsReduction(t *testing.T) { }) } } + +// Regression test for https://github.com/prometheus/prometheus/issues/17659 +func TestRemoteWriteHandler_ResponseStats(t *testing.T) { + payloadV1, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil, "snappy") + require.NoError(t, err) + payloadV2, _, _, err := buildV2WriteRequest(nil, writeV2RequestFixture.Timeseries, writeV2RequestFixture.Symbols, nil, nil, nil, "snappy") + require.NoError(t, err) + + for _, tt := range []struct { + msgType remoteapi.WriteMessageType + forceInjectHeaders bool + expectHeaders bool + }{ + { + msgType: remoteapi.WriteV1MessageType, + }, + { + msgType: remoteapi.WriteV1MessageType, + forceInjectHeaders: true, + expectHeaders: true, + }, + { + msgType: remoteapi.WriteV2MessageType, + expectHeaders: true, + }, + } { + t.Run(fmt.Sprintf("msg=%v/force-inject-headers=%v", tt.msgType, tt.forceInjectHeaders), func(t *testing.T) { + // Setup server side. + appendable := &mockAppendable{} + handler := NewWriteHandler( + promslog.NewNopLogger(), + nil, + appendable, + []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType, remoteapi.WriteV2MessageType}, + false, + false, + false, + ) + + if tt.forceInjectHeaders { + base := handler + handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + base.ServeHTTP(w, r) + + // Inject response header. This simulates PRWv1 server that uses PRWv2 response headers + // for confirmation of samples. This is not against spec and we support it. + w.Header().Set(rw20WrittenSamplesHeader, fmt.Sprintf("%d", len(appendable.samples))) + }) + } + + srv := httptest.NewServer(handler) + + // Send message and do the parse response flow. + c := &Client{Client: srv.Client(), urlString: srv.URL, timeout: 5 * time.Minute, writeProtoMsg: tt.msgType} + + payload := payloadV2 + if tt.msgType == remoteapi.WriteV1MessageType { + payload = payloadV1 + } + stats, err := c.Store(t.Context(), payload, 0) + require.NoError(t, err) + + fmt.Println(stats) + if tt.expectHeaders { + require.True(t, stats.Confirmed) + require.Equal(t, len(appendable.samples), stats.Samples) + } else { + require.False(t, stats.Confirmed) + } + }) + } +} From da253bddf507f54ff2cf452efb7201e9515f2676 Mon Sep 17 00:00:00 2001 From: bwplotka Date: Fri, 12 Dec 2025 22:47:03 +0000 Subject: [PATCH 2/5] fix: ensure remote PRWv1 write handler does not send false 0 response headers Signed-off-by: bwplotka --- go.mod | 4 ++-- go.sum | 8 ++++---- storage/remote/write_handler_test.go | 17 ++++++++--------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 9cf136eb39..35904d311b 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/hetznercloud/hcloud-go/v2 v2.29.0 github.com/ionos-cloud/sdk-go/v6 v6.3.4 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.18.1 + github.com/klauspost/compress v1.18.2 github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b github.com/linode/linodego v1.60.0 github.com/miekg/dns v1.1.68 @@ -56,7 +56,7 @@ require ( github.com/ovh/go-ovh v1.9.0 github.com/prometheus/alertmanager v0.28.1 github.com/prometheus/client_golang v1.23.2 - github.com/prometheus/client_golang/exp v0.0.0-20250914183048-a974e0d45e0a + github.com/prometheus/client_golang/exp v0.0.0-20251212205219-7ba246a648ca github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.67.4 github.com/prometheus/common/assets v0.2.0 diff --git a/go.sum b/go.sum index 579e86ca58..0aa41a311b 100644 --- a/go.sum +++ b/go.sum @@ -302,8 +302,8 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= -github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE= @@ -438,8 +438,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= -github.com/prometheus/client_golang/exp v0.0.0-20250914183048-a974e0d45e0a h1:RF1vfKM34/3DbGNis22BGd6sDDY3XBi0eM7pYqmOEO0= -github.com/prometheus/client_golang/exp v0.0.0-20250914183048-a974e0d45e0a/go.mod h1:FGJuwvfcPY0V5enm+w8zF1RNS062yugQtPPQp1c4Io4= +github.com/prometheus/client_golang/exp v0.0.0-20251212205219-7ba246a648ca h1:BOxmsLoL2ymn8lXJtorca7N/m+2vDQUDoEtPjf0iAxA= +github.com/prometheus/client_golang/exp v0.0.0-20251212205219-7ba246a648ca/go.mod h1:gndBHh3ZdjBozGcGrjUYjN3UJLRS3l2drALtu4lUt+k= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index 2610142db9..82cb000be7 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -1520,19 +1520,23 @@ func TestRemoteWriteHandler_ResponseStats(t *testing.T) { for _, tt := range []struct { msgType remoteapi.WriteMessageType + payload []byte forceInjectHeaders bool expectHeaders bool }{ { msgType: remoteapi.WriteV1MessageType, + payload: payloadV1, }, { msgType: remoteapi.WriteV1MessageType, + payload: payloadV1, forceInjectHeaders: true, expectHeaders: true, }, { msgType: remoteapi.WriteV2MessageType, + payload: payloadV2, expectHeaders: true, }, } { @@ -1552,11 +1556,11 @@ func TestRemoteWriteHandler_ResponseStats(t *testing.T) { if tt.forceInjectHeaders { base := handler handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - base.ServeHTTP(w, r) - // Inject response header. This simulates PRWv1 server that uses PRWv2 response headers // for confirmation of samples. This is not against spec and we support it. - w.Header().Set(rw20WrittenSamplesHeader, fmt.Sprintf("%d", len(appendable.samples))) + w.Header().Set(rw20WrittenSamplesHeader, "2") + + base.ServeHTTP(w, r) }) } @@ -1565,14 +1569,9 @@ func TestRemoteWriteHandler_ResponseStats(t *testing.T) { // Send message and do the parse response flow. c := &Client{Client: srv.Client(), urlString: srv.URL, timeout: 5 * time.Minute, writeProtoMsg: tt.msgType} - payload := payloadV2 - if tt.msgType == remoteapi.WriteV1MessageType { - payload = payloadV1 - } - stats, err := c.Store(t.Context(), payload, 0) + stats, err := c.Store(t.Context(), tt.payload, 0) require.NoError(t, err) - fmt.Println(stats) if tt.expectHeaders { require.True(t, stats.Confirmed) require.Equal(t, len(appendable.samples), stats.Samples) From 2e296c11ecdab0c96d2aecafbc221711e1aa9a8b Mon Sep 17 00:00:00 2001 From: bwplotka Date: Mon, 15 Dec 2025 15:41:53 +0000 Subject: [PATCH 3/5] chore: cut 3.8.1 Signed-off-by: bwplotka --- CHANGELOG.md | 6 +++++- VERSION | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3304339867..2da2d46a0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,13 @@ ## main / unreleased +## 3.8.1 / 2025-12-16 + +* [BUGFIX] remote: Fix Remote Write receiver, so it does not send wrong response headers for v1 flow and cause Prometheus senders to emit false partial error log and metrics. #17683 + ## 3.8.0 / 2025-11-28 -* [CHANGE] Remote-write 2 (receiving): Update to [2.0-rc.4 spec](https://github.com/prometheus/docs/blob/60c24e450010df38cfcb4f65df874f6f9b26dbcb/docs/specs/prw/remote_write_spec_2_0.md). "created timestamp" (CT) is now called "start timestamp" (ST). #17411 +* [CHANGE] remote: Update Remote Write receiving to [2.0-rc.4 spec](https://github.com/prometheus/docs/blob/60c24e450010df38cfcb4f65df874f6f9b26dbcb/docs/specs/prw/remote_write_spec_2_0.md). "created timestamp" (CT) is now called "start timestamp" (ST). #17411 * [CHANGE] TSDB: Native Histogram Custom Bounds with a NaN threshold are now rejected. #17287 * [FEATURE] OAuth2: support jwt-bearer grant-type (RFC7523 3.1). #17592 * [FEATURE] Dockerfile: Add OpenContainers spec labels to Dockerfile. #16483 diff --git a/VERSION b/VERSION index 19811903a7..f280719674 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.0 +3.8.1 From 7739353f5d8176ebc4c46668aea5de88518e3302 Mon Sep 17 00:00:00 2001 From: bwplotka Date: Mon, 15 Dec 2025 16:41:13 +0000 Subject: [PATCH 4/5] chore: upgrade npm Signed-off-by: bwplotka --- web/ui/mantine-ui/package.json | 4 ++-- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 4 ++-- web/ui/package.json | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/web/ui/mantine-ui/package.json b/web/ui/mantine-ui/package.json index 7ec13b1b8d..baf47d6f6b 100644 --- a/web/ui/mantine-ui/package.json +++ b/web/ui/mantine-ui/package.json @@ -1,7 +1,7 @@ { "name": "@prometheus-io/mantine-ui", "private": true, - "version": "0.308.0", + "version": "0.308.1", "type": "module", "scripts": { "start": "vite", @@ -28,7 +28,7 @@ "@microsoft/fetch-event-source": "^2.0.1", "@nexucis/fuzzy": "^0.5.1", "@nexucis/kvsearch": "^0.9.1", - "@prometheus-io/codemirror-promql": "0.308.0", + "@prometheus-io/codemirror-promql": "0.308.1", "@reduxjs/toolkit": "^2.10.1", "@tabler/icons-react": "^3.35.0", "@tanstack/react-query": "^5.90.7", diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index ee7bcc045f..5f632320bd 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.308.0", + "version": "0.308.1", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "0.308.0", + "@prometheus-io/lezer-promql": "0.308.1", "lru-cache": "^11.2.2" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index 034ead9741..85cc4c50ed 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.308.0", + "version": "0.308.1", "description": "lezer-based PromQL grammar", "main": "dist/index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index 7f2961784b..883ee7aaee 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "prometheus-io", - "version": "0.308.0", + "version": "0.308.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "prometheus-io", - "version": "0.308.0", + "version": "0.308.1", "workspaces": [ "mantine-ui", "module/*" diff --git a/web/ui/package.json b/web/ui/package.json index 5023d1d21b..44d0b52ce0 100644 --- a/web/ui/package.json +++ b/web/ui/package.json @@ -1,7 +1,7 @@ { "name": "prometheus-io", "description": "Monorepo for the Prometheus UI", - "version": "0.308.0", + "version": "0.308.1", "private": true, "scripts": { "build": "bash build_ui.sh --all", From bf552e66c07b722e16cef43bf8584580e6d3cda4 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 15 Dec 2025 20:41:23 +0000 Subject: [PATCH 5/5] Merge pull request #17695 from roidelapluie/roidelapluie/fixfuzzing chore: Update OSS-Fuzz CIFuzz actions to latest version --- .github/workflows/fuzzing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index 3d3aa82d1c..24702c2920 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -10,12 +10,12 @@ jobs: steps: - name: Build Fuzzers id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@cafd7a0eb8ecb4e007c56897996a9b65c49c972f # master + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@537c8005ba4c9de026b2fa3550663280d25d6175 # master with: oss-fuzz-project-name: "prometheus" dry-run: false - name: Run Fuzzers - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@cafd7a0eb8ecb4e007c56897996a9b65c49c972f # master + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@537c8005ba4c9de026b2fa3550663280d25d6175 # master # Note: Regularly check for updates to the pinned commit hash at: # https://github.com/google/oss-fuzz/tree/master/infra/cifuzz/actions/run_fuzzers with: