diff --git a/go.mod b/go.mod index 9cf136eb39..35904d311b 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/hetznercloud/hcloud-go/v2 v2.29.0 github.com/ionos-cloud/sdk-go/v6 v6.3.4 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.18.1 + github.com/klauspost/compress v1.18.2 github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b github.com/linode/linodego v1.60.0 github.com/miekg/dns v1.1.68 @@ -56,7 +56,7 @@ require ( github.com/ovh/go-ovh v1.9.0 github.com/prometheus/alertmanager v0.28.1 github.com/prometheus/client_golang v1.23.2 - github.com/prometheus/client_golang/exp v0.0.0-20250914183048-a974e0d45e0a + github.com/prometheus/client_golang/exp v0.0.0-20251212205219-7ba246a648ca github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.67.4 github.com/prometheus/common/assets v0.2.0 diff --git a/go.sum b/go.sum index 579e86ca58..0aa41a311b 100644 --- a/go.sum +++ b/go.sum @@ -302,8 +302,8 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= -github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE= @@ -438,8 +438,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= -github.com/prometheus/client_golang/exp v0.0.0-20250914183048-a974e0d45e0a h1:RF1vfKM34/3DbGNis22BGd6sDDY3XBi0eM7pYqmOEO0= -github.com/prometheus/client_golang/exp v0.0.0-20250914183048-a974e0d45e0a/go.mod h1:FGJuwvfcPY0V5enm+w8zF1RNS062yugQtPPQp1c4Io4= +github.com/prometheus/client_golang/exp v0.0.0-20251212205219-7ba246a648ca h1:BOxmsLoL2ymn8lXJtorca7N/m+2vDQUDoEtPjf0iAxA= +github.com/prometheus/client_golang/exp v0.0.0-20251212205219-7ba246a648ca/go.mod h1:gndBHh3ZdjBozGcGrjUYjN3UJLRS3l2drALtu4lUt+k= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/storage/remote/client.go b/storage/remote/client.go index c535ea3425..0f2b5ddca6 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -301,6 +301,9 @@ func (c *Client) Store(ctx context.Context, req []byte, attempt int) (WriteRespo _ = httpResp.Body.Close() }() + // NOTE(bwplotka): Only PRW2 spec defines response HTTP headers. However, spec does not block + // PRW1 from sending them too for reliability. Support this case. + // // TODO(bwplotka): Pass logger and emit debug on error? // Parsing error means there were some response header values we can't parse, // we can continue handling. diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index f8296b4a80..2bc65e8286 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -96,6 +96,10 @@ func isHistogramValidationError(err error) bool { } // Store implements remoteapi.writeStorage interface. +// TODO(bwplotka): Improve remoteapi.Store API. Right now it's confusing if PRWv1 flows should use WriteResponse or not. +// If it's not filled, it will be "confirmed zero" which caused partial error reporting on client side in the past. +// Temporary fix was done to only care about WriteResponse stats for PRW2 (see https://github.com/prometheus/client_golang/pull/1927 +// but better approach would be to only confirm if explicit stats were injected. func (h *writeHandler) Store(r *http.Request, msgType remoteapi.WriteMessageType) (*remoteapi.WriteResponse, error) { // Store receives request with decompressed content in body. body, err := io.ReadAll(r.Body) diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index f1c064c64d..82cb000be7 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -1510,3 +1510,74 @@ func TestHistogramsReduction(t *testing.T) { }) } } + +// Regression test for https://github.com/prometheus/prometheus/issues/17659 +func TestRemoteWriteHandler_ResponseStats(t *testing.T) { + payloadV1, _, _, err := buildWriteRequest(nil, writeRequestFixture.Timeseries, nil, nil, nil, nil, "snappy") + require.NoError(t, err) + payloadV2, _, _, err := buildV2WriteRequest(nil, writeV2RequestFixture.Timeseries, writeV2RequestFixture.Symbols, nil, nil, nil, "snappy") + require.NoError(t, err) + + for _, tt := range []struct { + msgType remoteapi.WriteMessageType + payload []byte + forceInjectHeaders bool + expectHeaders bool + }{ + { + msgType: remoteapi.WriteV1MessageType, + payload: payloadV1, + }, + { + msgType: remoteapi.WriteV1MessageType, + payload: payloadV1, + forceInjectHeaders: true, + expectHeaders: true, + }, + { + msgType: remoteapi.WriteV2MessageType, + payload: payloadV2, + expectHeaders: true, + }, + } { + t.Run(fmt.Sprintf("msg=%v/force-inject-headers=%v", tt.msgType, tt.forceInjectHeaders), func(t *testing.T) { + // Setup server side. + appendable := &mockAppendable{} + handler := NewWriteHandler( + promslog.NewNopLogger(), + nil, + appendable, + []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType, remoteapi.WriteV2MessageType}, + false, + false, + false, + ) + + if tt.forceInjectHeaders { + base := handler + handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Inject response header. This simulates PRWv1 server that uses PRWv2 response headers + // for confirmation of samples. This is not against spec and we support it. + w.Header().Set(rw20WrittenSamplesHeader, "2") + + base.ServeHTTP(w, r) + }) + } + + srv := httptest.NewServer(handler) + + // Send message and do the parse response flow. + c := &Client{Client: srv.Client(), urlString: srv.URL, timeout: 5 * time.Minute, writeProtoMsg: tt.msgType} + + stats, err := c.Store(t.Context(), tt.payload, 0) + require.NoError(t, err) + + if tt.expectHeaders { + require.True(t, stats.Confirmed) + require.Equal(t, len(appendable.samples), stats.Samples) + } else { + require.False(t, stats.Confirmed) + } + }) + } +}