upstream/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2026-06-08 16:12:16 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

prompb: Generate client proto using Opaque API, for parsing only.

Browse source 
Signed-off-by: bwplotka <bwplotka@gmail.com>
This commit is contained in:
bwplotka 2024-12-29 16:25:40 +00:00
parent 281306765e
commit 9ad26bb581
11 changed files with 2494 additions and 4459 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
model/textparse/nhcbparse_test.go
View file

@ -15,11 +15,9 @@ package textparse
import (
"bytes"
"encoding/binary"
"strconv"
"testing"
"github.com/gogo/protobuf/proto"
"github.com/stretchr/testify/require"
"github.com/prometheus/common/model"
@ -27,7 +25,7 @@ import (
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
clientv1 "github.com/prometheus/prometheus/prompb/io/prometheus/client/v1"
)
func TestNHCBParserOnOMParser(t *testing.T) {
@ -894,23 +892,10 @@ metric: <
>
`}
varintBuf := make([]byte, binary.MaxVarintLen32)
buf := &bytes.Buffer{}
for _, tmf := range testMetricFamilies {
pb := &dto.MetricFamily{}
// From text to proto message.
require.NoError(t, proto.UnmarshalText(tmf, pb))
// From proto message to binary protobuf.
protoBuf, err := proto.Marshal(pb)
require.NoError(t, err)
// Write first length, then binary protobuf.
varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf)))
buf.Write(varintBuf[:varintLength])
buf.Write(protoBuf)
require.NoError(t, clientv1.MetricFamilyFromTextToDelimitedProto([]byte(tmf), buf))
}
return buf
}

97
model/textparse/protobufparse.go
View file

@ -25,15 +25,16 @@ import (
"sync"
"unicode/utf8"
"github.com/gogo/protobuf/proto"
"github.com/gogo/protobuf/types"
"github.com/prometheus/common/model"
"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
clientv1 "github.com/prometheus/prometheus/prompb/io/prometheus/client/v1"
)
// floatFormatBufPool is exclusively used in formatOpenMetricsFloat.
@ -80,7 +81,7 @@ type ProtobufParser struct {
builder labels.ScratchBuilder // held here to reduce allocations when building Labels
mf *dto.MetricFamily
mf *clientv1.MetricFamily
// Whether to also parse a classic histogram that is also present as a
// native histogram.
@ -95,7 +96,7 @@ func NewProtobufParser(b []byte, parseClassicHistograms bool, st *labels.SymbolT
return &ProtobufParser{
in: b,
state: EntryInvalid,
mf: &dto.MetricFamily{},
mf: &clientv1.MetricFamily{},
metricBytes: &bytes.Buffer{},
parseClassicHistograms: parseClassicHistograms,
builder: labels.NewScratchBuilderWithSymbolTable(st, 16),
@ -111,13 +112,13 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) {
v float64
)
switch p.mf.GetType() {
case dto.MetricType_COUNTER:
case clientv1.MetricType_COUNTER:
v = m.GetCounter().GetValue()
case dto.MetricType_GAUGE:
case clientv1.MetricType_GAUGE:
v = m.GetGauge().GetValue()
case dto.MetricType_UNTYPED:
case clientv1.MetricType_UNTYPED:
v = m.GetUntyped().GetValue()
case dto.MetricType_SUMMARY:
case clientv1.MetricType_SUMMARY:
s := m.GetSummary()
switch p.fieldPos {
case -2:
@ -131,7 +132,7 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) {
default:
v = s.GetQuantile()[p.fieldPos].GetValue()
}
case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM:
case clientv1.MetricType_HISTOGRAM, clientv1.MetricType_GAUGE_HISTOGRAM:
// This should only happen for a classic histogram.
h := m.GetHistogram()
switch p.fieldPos {
@ -214,7 +215,7 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his
fh.NegativeSpans[i].Offset = span.GetOffset()
fh.NegativeSpans[i].Length = span.GetLength()
}
if p.mf.GetType() == dto.MetricType_GAUGE_HISTOGRAM {
if p.mf.GetType() == clientv1.MetricType_GAUGE_HISTOGRAM {
fh.CounterResetHint = histogram.GaugeType
}
fh.Compact(0)
@ -246,7 +247,7 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his
sh.NegativeSpans[i].Offset = span.GetOffset()
sh.NegativeSpans[i].Length = span.GetLength()
}
if p.mf.GetType() == dto.MetricType_GAUGE_HISTOGRAM {
if p.mf.GetType() == clientv1.MetricType_GAUGE_HISTOGRAM {
sh.CounterResetHint = histogram.GaugeType
}
sh.Compact(0)
@ -269,15 +270,15 @@ func (p *ProtobufParser) Help() ([]byte, []byte) {
func (p *ProtobufParser) Type() ([]byte, model.MetricType) {
n := p.metricBytes.Bytes()
switch p.mf.GetType() {
case dto.MetricType_COUNTER:
case clientv1.MetricType_COUNTER:
return n, model.MetricTypeCounter
case dto.MetricType_GAUGE:
case clientv1.MetricType_GAUGE:
return n, model.MetricTypeGauge
case dto.MetricType_HISTOGRAM:
case clientv1.MetricType_HISTOGRAM:
return n, model.MetricTypeHistogram
case dto.MetricType_GAUGE_HISTOGRAM:
case clientv1.MetricType_GAUGE_HISTOGRAM:
return n, model.MetricTypeGaugeHistogram
case dto.MetricType_SUMMARY:
case clientv1.MetricType_SUMMARY:
return n, model.MetricTypeSummary
}
return n, model.MetricTypeUnknown
@ -327,11 +328,11 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool {
return false
}
m := p.mf.GetMetric()[p.metricPos]
var exProto *dto.Exemplar
var exProto *clientv1.Exemplar
switch p.mf.GetType() {
case dto.MetricType_COUNTER:
case clientv1.MetricType_COUNTER:
exProto = m.GetCounter().GetExemplar()
case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM:
case clientv1.MetricType_HISTOGRAM, clientv1.MetricType_GAUGE_HISTOGRAM:
isClassic := p.state == EntrySeries
if !isClassic && len(m.GetHistogram().GetExemplars()) > 0 {
exs := m.GetHistogram().GetExemplars()
@ -393,22 +394,20 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool {
// CreatedTimestamp returns CT or nil if CT is not present or
// invalid (as timestamp e.g. negative value) on counters, summaries or histograms.
func (p *ProtobufParser) CreatedTimestamp() *int64 {
var ct *types.Timestamp
var ct *timestamppb.Timestamp
switch p.mf.GetType() {
case dto.MetricType_COUNTER:
case clientv1.MetricType_COUNTER:
ct = p.mf.GetMetric()[p.metricPos].GetCounter().GetCreatedTimestamp()
case dto.MetricType_SUMMARY:
case clientv1.MetricType_SUMMARY:
ct = p.mf.GetMetric()[p.metricPos].GetSummary().GetCreatedTimestamp()
case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM:
case clientv1.MetricType_HISTOGRAM, clientv1.MetricType_GAUGE_HISTOGRAM:
ct = p.mf.GetMetric()[p.metricPos].GetHistogram().GetCreatedTimestamp()
default:
}
ctAsTime, err := types.TimestampFromProto(ct)
if err != nil {
// Errors means ct == nil or invalid timestamp, which we silently ignore.
if ct == nil {
return nil
}
ctMilis := ctAsTime.UnixMilli()
ctMilis := ct.AsTime().UnixMilli()
return &ctMilis
}
@ -443,19 +442,19 @@ func (p *ProtobufParser) Next() (Entry, error) {
return EntryInvalid, fmt.Errorf("invalid help for metric %q: %s", name, help)
}
switch p.mf.GetType() {
case dto.MetricType_COUNTER,
dto.MetricType_GAUGE,
dto.MetricType_HISTOGRAM,
dto.MetricType_GAUGE_HISTOGRAM,
dto.MetricType_SUMMARY,
dto.MetricType_UNTYPED:
case clientv1.MetricType_COUNTER,
clientv1.MetricType_GAUGE,
clientv1.MetricType_HISTOGRAM,
clientv1.MetricType_GAUGE_HISTOGRAM,
clientv1.MetricType_SUMMARY,
clientv1.MetricType_UNTYPED:
// All good.
default:
return EntryInvalid, fmt.Errorf("unknown metric type for metric %q: %s", name, p.mf.GetType())
}
unit := p.mf.GetUnit()
if len(unit) > 0 {
if p.mf.GetType() == dto.MetricType_COUNTER && strings.HasSuffix(name, "_total") {
if p.mf.GetType() == clientv1.MetricType_COUNTER && strings.HasSuffix(name, "_total") {
if !strings.HasSuffix(name[:len(name)-6], unit) || len(name)-6 < len(unit)+1 || name[len(name)-6-len(unit)-1] != '_' {
return EntryInvalid, fmt.Errorf("unit %q not a suffix of counter %q", unit, name)
}
@ -468,7 +467,7 @@ func (p *ProtobufParser) Next() (Entry, error) {
p.state = EntryHelp
case EntryHelp:
if p.mf.Unit != "" {
if p.mf.GetUnit() != "" {
p.state = EntryUnit
} else {
p.state = EntryType
@ -477,7 +476,7 @@ func (p *ProtobufParser) Next() (Entry, error) {
p.state = EntryType
case EntryType:
t := p.mf.GetType()
if (t == dto.MetricType_HISTOGRAM || t == dto.MetricType_GAUGE_HISTOGRAM) &&
if (t == clientv1.MetricType_HISTOGRAM || t == clientv1.MetricType_GAUGE_HISTOGRAM) &&
isNativeHistogram(p.mf.GetMetric()[0].GetHistogram()) {
p.state = EntryHistogram
} else {
@ -495,9 +494,9 @@ func (p *ProtobufParser) Next() (Entry, error) {
}
t := p.mf.GetType()
if p.state == EntrySeries && !p.fieldsDone &&
(t == dto.MetricType_SUMMARY ||
t == dto.MetricType_HISTOGRAM ||
t == dto.MetricType_GAUGE_HISTOGRAM) {
(t == clientv1.MetricType_SUMMARY ||
t == clientv1.MetricType_HISTOGRAM ||
t == clientv1.MetricType_GAUGE_HISTOGRAM) {
p.fieldPos++
} else {
p.metricPos++
@ -508,7 +507,7 @@ func (p *ProtobufParser) Next() (Entry, error) {
// histograms, we have to switch back to native
// histograms after parsing a classic histogram.
if p.state == EntrySeries &&
(t == dto.MetricType_HISTOGRAM || t == dto.MetricType_GAUGE_HISTOGRAM) &&
(t == clientv1.MetricType_HISTOGRAM || t == clientv1.MetricType_GAUGE_HISTOGRAM) &&
isNativeHistogram(p.mf.GetMetric()[0].GetHistogram()) {
p.state = EntryHistogram
}
@ -558,7 +557,7 @@ func (p *ProtobufParser) updateMetricBytes() error {
// state.
func (p *ProtobufParser) getMagicName() string {
t := p.mf.GetType()
if p.state == EntryHistogram || (t != dto.MetricType_HISTOGRAM && t != dto.MetricType_GAUGE_HISTOGRAM && t != dto.MetricType_SUMMARY) {
if p.state == EntryHistogram || (t != clientv1.MetricType_HISTOGRAM && t != clientv1.MetricType_GAUGE_HISTOGRAM && t != clientv1.MetricType_SUMMARY) {
return p.mf.GetName()
}
if p.fieldPos == -2 {
@ -567,7 +566,7 @@ func (p *ProtobufParser) getMagicName() string {
if p.fieldPos == -1 {
return p.mf.GetName() + "_sum"
}
if t == dto.MetricType_HISTOGRAM || t == dto.MetricType_GAUGE_HISTOGRAM {
if t == clientv1.MetricType_HISTOGRAM || t == clientv1.MetricType_GAUGE_HISTOGRAM {
return p.mf.GetName() + "_bucket"
}
return p.mf.GetName()
@ -580,12 +579,12 @@ func (p *ProtobufParser) getMagicLabel() (bool, string, string) {
return false, "", ""
}
switch p.mf.GetType() {
case dto.MetricType_SUMMARY:
case clientv1.MetricType_SUMMARY:
qq := p.mf.GetMetric()[p.metricPos].GetSummary().GetQuantile()
q := qq[p.fieldPos]
p.fieldsDone = p.fieldPos == len(qq)-1
return true, model.QuantileLabel, formatOpenMetricsFloat(q.GetQuantile())
case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM:
case clientv1.MetricType_HISTOGRAM, clientv1.MetricType_GAUGE_HISTOGRAM:
bb := p.mf.GetMetric()[p.metricPos].GetHistogram().GetBucket()
if p.fieldPos >= len(bb) {
p.fieldsDone = true
@ -605,11 +604,11 @@ var errInvalidVarint = errors.New("protobufparse: invalid varint encountered")
// specific to a MetricFamily, utilizes the more efficient gogo-protobuf
// unmarshaling, and acts on a byte slice directly without any additional
// staging buffers.
func readDelimited(b []byte, mf *dto.MetricFamily) (n int, err error) {
func readDelimited(b []byte, mf *clientv1.MetricFamily) (n int, err error) {
if len(b) == 0 {
return 0, io.EOF
}
messageLength, varIntLength := proto.DecodeVarint(b)
messageLength, varIntLength := protowire.ConsumeVarint(b)
if varIntLength == 0 || varIntLength > binary.MaxVarintLen32 {
return 0, errInvalidVarint
}
@ -618,7 +617,7 @@ func readDelimited(b []byte, mf *dto.MetricFamily) (n int, err error) {
return 0, fmt.Errorf("protobufparse: insufficient length of buffer, expected at least %d bytes, got %d bytes", totalLength, len(b))
}
mf.Reset()
return totalLength, mf.Unmarshal(b[varIntLength:totalLength])
return totalLength, proto.Unmarshal(b[varIntLength:totalLength], mf)
}
// formatOpenMetricsFloat works like the usual Go string formatting of a float
@ -659,7 +658,7 @@ func formatOpenMetricsFloat(f float64) string {
// zero) to signal that the histogram is meant to be parsed as a native
// histogram. Failing to do so will cause Prometheus to parse it as a classic
// histogram as long as no observations have happened.
func isNativeHistogram(h *dto.Histogram) bool {
func isNativeHistogram(h *clientv1.Histogram) bool {
return len(h.GetPositiveSpan()) > 0 ||
len(h.GetNegativeSpan()) > 0 ||
h.GetZeroThreshold() > 0 ||

19
model/textparse/protobufparse_test.go
View file

@ -15,17 +15,15 @@ package textparse
import (
"bytes"
"encoding/binary"
"testing"
"github.com/gogo/protobuf/proto"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
clientv1 "github.com/prometheus/prometheus/prompb/io/prometheus/client/v1"
)
func createTestProtoBuf(t testing.TB) *bytes.Buffer {
@ -803,23 +801,10 @@ metric: <
`,
}
varintBuf := make([]byte, binary.MaxVarintLen32)
buf := &bytes.Buffer{}
for _, tmf := range testMetricFamilies {
pb := &dto.MetricFamily{}
// From text to proto message.
require.NoError(t, proto.UnmarshalText(tmf, pb))
// From proto message to binary protobuf.
protoBuf, err := proto.Marshal(pb)
require.NoError(t, err)
// Write first length, then binary protobuf.
varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf)))
buf.Write(varintBuf[:varintLength])
buf.Write(protoBuf)
require.NoError(t, clientv1.MetricFamilyFromTextToDelimitedProto([]byte(tmf), buf))
}
return buf
}

11
prompb/buf.gen.yaml Normal file
View file

@ -0,0 +1,11 @@
version: v2
plugins:
- remote: buf.build/protocolbuffers/go:v1.36.1
out: .
opt:
- Mio/prometheus/client/v1/metrics.proto=io/prometheus/client/v1
- default_api_level=API_OPAQUE
inputs:
- directory: . # Pointing to the module in the buf.yaml.

5
prompb/buf.yaml
View file

@ -5,17 +5,16 @@ lint:
ENUM_VALUE_PREFIX:
- remote.proto
- types.proto
- io/prometheus/client/metrics.proto
- io/prometheus/client/v1/metrics.proto
ENUM_ZERO_VALUE_SUFFIX:
- remote.proto
- types.proto
- io/prometheus/client/metrics.proto
- io/prometheus/client/v1/metrics.proto
PACKAGE_DIRECTORY_MATCH:
- remote.proto
- types.proto
PACKAGE_VERSION_SUFFIX:
- remote.proto
- types.proto
- io/prometheus/client/metrics.proto
deps:
- buf.build/gogo/protobuf

4300
prompb/io/prometheus/client/metrics.pb.go
View file

File diff suppressed because it is too large Load diff

43
prompb/io/prometheus/client/v1/custom.go Normal file
View file

@ -0,0 +1,43 @@
// Copyright 2024 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clientv1
import (
"io"
"google.golang.org/protobuf/encoding/prototext"
"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/proto"
)
func MetricFamilyFromTextToDelimitedProto(text []byte, w io.Writer) error {
mf := &MetricFamily{}
if err := prototext.Unmarshal(text, mf); err != nil {
return err
}
// From proto message to binary protobuf.
protoBuf, err := proto.Marshal(mf)
if err != nil {
return err
}
// Write first length, then binary protobuf.
varintBuf := protowire.AppendVarint(nil, uint64(len(protoBuf)))
if _, err := w.Write(varintBuf); err != nil {
return err
}
if _, err := w.Write(protoBuf); err != nil {
return err
}
return nil
}

2337
prompb/io/prometheus/client/v1/metrics.pb.go Normal file
View file

File diff suppressed because it is too large Load diff

48
prompb/io/prometheus/client/metrics.proto → prompb/io/prometheus/client/v1/metrics.proto
View file

@ -1,4 +1,4 @@
// Copyright 2013 Prometheus Team
// Copyright 2024 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@ -13,15 +13,19 @@
// This is copied and lightly edited from
// github.com/prometheus/client_model/io/prometheus/client/metrics.proto
// and finally converted to proto3 syntax to make it usable for the
// gogo-protobuf approach taken within prometheus/prometheus.
// and finally converted to proto3 syntax. Initially to make it usable for the
// gogo-protobuf approach taken within prometheus/prometheus, but now this copy
// is actually source of the truth for Prometheus protobuf 1.0 encoding.
//
// From now client_model is deprecated and used only for dto, client_golang
// needs for v1 compatibility (due to opaque API breaking it).
syntax = "proto3";
edition = "2023";
package io.prometheus.client;
option go_package = "io_prometheus_client";
package io.prometheus.client.v1;
option go_package = "clientv1";
import "gogoproto/gogo.proto";
import "google/protobuf/timestamp.proto";
message LabelPair {
@ -50,7 +54,7 @@ message Gauge {
message Counter {
double value = 1;
Exemplar exemplar = 2;
Exemplar exemplar = 2 [lazy = true];
google.protobuf.Timestamp created_timestamp = 3;
}
@ -63,7 +67,7 @@ message Quantile {
message Summary {
uint64 sample_count = 1;
double sample_sum = 2;
repeated Quantile quantile = 3 [(gogoproto.nullable) = false];
repeated Quantile quantile = 3;
google.protobuf.Timestamp created_timestamp = 4;
}
@ -77,7 +81,7 @@ message Histogram {
double sample_count_float = 4; // Overrides sample_count if > 0.
double sample_sum = 2;
// Buckets for the classic histogram.
repeated Bucket bucket = 3 [(gogoproto.nullable) = false]; // Ordered in increasing order of upper_bound, +Inf bucket is optional.
repeated Bucket bucket = 3; // Ordered in increasing order of upper_bound, +Inf bucket is optional.
google.protobuf.Timestamp created_timestamp = 15;
@ -95,7 +99,7 @@ message Histogram {
double zero_count_float = 8; // Overrides sb_zero_count if > 0.
// Negative buckets for the native histogram.
repeated BucketSpan negative_span = 9 [(gogoproto.nullable) = false];
repeated BucketSpan negative_span = 9;
// Use either "negative_delta" or "negative_count", the former for
// regular histograms with integer counts, the latter for float
// histograms.
@ -106,7 +110,7 @@ message Histogram {
// Use a no-op span (offset 0, length 0) for a native histogram without any
// observations yet and with a zero_threshold of 0. Otherwise, it would be
// indistinguishable from a classic histogram.
repeated BucketSpan positive_span = 12 [(gogoproto.nullable) = false];
repeated BucketSpan positive_span = 12;
// Use either "positive_delta" or "positive_count", the former for
// regular histograms with integer counts, the latter for float
// histograms.
@ -114,14 +118,14 @@ message Histogram {
repeated double positive_count = 14; // Absolute count of each bucket.
// Only used for native histograms. These exemplars MUST have a timestamp.
repeated Exemplar exemplars = 16;
repeated Exemplar exemplars = 16 [lazy = true];
}
message Bucket {
uint64 cumulative_count = 1; // Cumulative in increasing order.
double cumulative_count_float = 4; // Overrides cumulative_count if > 0.
double upper_bound = 2; // Inclusive.
Exemplar exemplar = 3;
Exemplar exemplar = 3 [lazy = true];
}
// A BucketSpan defines a number of consecutive buckets in a native
@ -136,18 +140,18 @@ message BucketSpan {
}
message Exemplar {
repeated LabelPair label = 1 [(gogoproto.nullable) = false];
repeated LabelPair label = 1;
double value = 2;
google.protobuf.Timestamp timestamp = 3; // OpenMetrics-style.
}
message Metric {
repeated LabelPair label = 1 [(gogoproto.nullable) = false];
Gauge gauge = 2;
Counter counter = 3;
Summary summary = 4;
Untyped untyped = 5;
Histogram histogram = 7;
repeated LabelPair label = 1 [lazy = true];
Gauge gauge = 2 [lazy = true];
Counter counter = 3 [lazy = true];
Summary summary = 4 [lazy = true];
Untyped untyped = 5 [lazy = true];
Histogram histogram = 7 [lazy = true];
int64 timestamp_ms = 6;
}
@ -155,6 +159,6 @@ message MetricFamily {
string name = 1;
string help = 2;
MetricType type = 3;
repeated Metric metric = 4 [(gogoproto.nullable) = false];
repeated Metric metric = 4 [lazy = true];
string unit = 5;
}

41
scrape/scrape_test.go
View file

@ -32,7 +32,6 @@ import (
"text/template"
"time"
"github.com/gogo/protobuf/proto"
"github.com/google/go-cmp/cmp"
"github.com/grafana/regexp"
"github.com/prometheus/client_golang/prometheus"
@ -43,6 +42,9 @@ import (
"github.com/prometheus/common/model"
"github.com/prometheus/common/promslog"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
clientv1 "github.com/prometheus/prometheus/prompb/io/prometheus/client/v1"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery"
@ -2486,7 +2488,7 @@ metric: <
buf := &bytes.Buffer{}
if test.contentType == "application/vnd.google.protobuf" {
require.NoError(t, textToProto(test.scrapeText, buf))
require.NoError(t, clientv1.MetricFamilyFromTextToDelimitedProto([]byte(test.scrapeText), buf))
} else {
buf.WriteString(test.scrapeText)
}
@ -2501,26 +2503,6 @@ metric: <
}
}
func textToProto(text string, buf *bytes.Buffer) error {
// In case of protobuf, we have to create the binary representation.
pb := &dto.MetricFamily{}
// From text to proto message.
err := proto.UnmarshalText(text, pb)
if err != nil {
return err
}
// From proto message to binary protobuf.
protoBuf, err := proto.Marshal(pb)
if err != nil {
return err
}
// Write first length, then binary protobuf.
varintBuf := binary.AppendUvarint(nil, uint64(len(protoBuf)))
buf.Write(varintBuf)
buf.Write(protoBuf)
return nil
}
func TestScrapeLoopAppendExemplarSeries(t *testing.T) {
scrapeText := []string{`metric_total{n="1"} 1 # {t="1"} 1.0 10000
# EOF`, `metric_total{n="1"} 2 # {t="2"} 2.0 20000
@ -4201,20 +4183,7 @@ metric: <
if metricsText.contentType != "" {
w.Header().Set("Content-Type", `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`)
for _, text := range metricsText.text {
buf := &bytes.Buffer{}
// In case of protobuf, we have to create the binary representation.
pb := &dto.MetricFamily{}
// From text to proto message.
require.NoError(t, proto.UnmarshalText(text, pb))
// From proto message to binary protobuf.
protoBuf, err := proto.Marshal(pb)
require.NoError(t, err)
// Write first length, then binary protobuf.
varintBuf := binary.AppendUvarint(nil, uint64(len(protoBuf)))
buf.Write(varintBuf)
buf.Write(protoBuf)
w.Write(buf.Bytes())
require.NoError(t, clientv1.MetricFamilyFromTextToDelimitedProto([]byte(text), w))
}
} else {
for _, text := range metricsText.text {

33
scripts/genproto.sh
View file

@ -10,27 +10,33 @@ if ! [[ "$0" =~ "scripts/genproto.sh" ]]; then
exit 255
fi
# TODO(bwplotka): Move to buf, this is not OSS agnostic, likely won't work locally.
if ! [[ $(protoc --version) =~ "3.15.8" ]]; then
echo "could not find protoc 3.15.8, is it installed + in PATH? Consider commenting out this check for local flow"
exit 255
fi
SED=$(which gsed 2>/dev/null || which sed)
# Since we run go install, go mod download, the go.sum will change.
# Make a backup.
cp go.sum go.sum.bak
INSTALL_PKGS="golang.org/x/tools/cmd/goimports github.com/gogo/protobuf/protoc-gen-gogofast github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger"
INSTALL_PKGS="golang.org/x/tools/cmd/goimports github.com/gogo/protobuf/protoc-gen-gogofast github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger github.com/bufbuild/buf/cmd/buf@v1.48.0"
for pkg in ${INSTALL_PKGS}; do
GO111MODULE=on go install "$pkg"
done
# Build client proto via buf.
pushd ./prompb
buf generate --path=./io/prometheus/client/v1
popd
# TODO(bwplotka): Move write building to buf e.g. once moved out of gogo.
if ! [[ $(protoc --version) =~ "3.15.8" ]]; then
echo "could not find protoc 3.15.8, is it installed + in PATH? Consider commenting out this check for local flow"
exit 255
fi
PROM_ROOT="${PWD}"
PROM_PATH="${PROM_ROOT}/prompb"
GOGOPROTO_ROOT="$(GO111MODULE=on go list -mod=readonly -f '{{ .Dir }}' -m github.com/gogo/protobuf)"
GOGOPROTO_PATH="${GOGOPROTO_ROOT}:${GOGOPROTO_ROOT}/protobuf"
GRPC_GATEWAY_ROOT="$(GO111MODULE=on go list -mod=readonly -f '{{ .Dir }}' -m github.com/grpc-ecosystem/grpc-gateway)"
DIRS="prompb"
echo "generating code"
@ -44,14 +50,11 @@ for dir in ${DIRS}; do
protoc --gogofast_out=plugins=grpc:. -I=. \
-I="${GOGOPROTO_PATH}" \
./io/prometheus/write/v2/*.proto
protoc --gogofast_out=Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,paths=source_relative:. -I=. \
-I="${GOGOPROTO_PATH}" \
./io/prometheus/client/*.proto
sed -i.bak -E 's/import _ \"github.com\/gogo\/protobuf\/gogoproto\"//g' *.pb.go
sed -i.bak -E 's/import _ \"google\/protobuf\"//g' *.pb.go
sed -i.bak -E 's/\t_ \"google\/protobuf\"//g' *.pb.go
sed -i.bak -E 's/golang\/protobuf\/descriptor/gogo\/protobuf\/protoc-gen-gogo\/descriptor/g' *.go
sed -i.bak -E 's/golang\/protobuf/gogo\/protobuf/g' *.go
${SED} -i.bak -E 's/import _ \"github.com\/gogo\/protobuf\/gogoproto\"//g' *.pb.go
${SED} -i.bak -E 's/import _ \"google\/protobuf\"//g' *.pb.go
${SED} -i.bak -E 's/\t_ \"google\/protobuf\"//g' *.pb.go
${SED} -i.bak -E 's/golang\/protobuf\/descriptor/gogo\/protobuf\/protoc-gen-gogo\/descriptor/g' *.go
${SED} -i.bak -E 's/golang\/protobuf/gogo\/protobuf/g' *.go
rm -f -- *.bak
goimports -w ./*.go ./io/prometheus/client/*.go
popd