From ee8a429af3917e8a5fbb2eea51eeffae96d311a7 Mon Sep 17 00:00:00 2001 From: Jesus Vazquez Date: Thu, 17 Oct 2024 15:49:40 +0200 Subject: [PATCH] Refactor Append interface to receive hints pointer Includes both: - Extending AppendHistogram with AppendHints - Extending AppendHistogramCTZeroSample with AppendHints Signed-off-by: Jesus Vazquez --- cmd/prometheus/main.go | 6 +- cmd/prometheus/main_test.go | 6 +- cmd/promtool/backfill.go | 2 +- cmd/promtool/rules.go | 2 +- cmd/promtool/tsdb.go | 2 +- promql/bench_test.go | 8 +- promql/engine_test.go | 20 +-- promql/functions_test.go | 2 +- promql/promqltest/test.go | 4 +- rules/group.go | 8 +- rules/manager_test.go | 20 +-- scrape/helpers_test.go | 22 +-- scrape/scrape.go | 14 +- scrape/scrape_test.go | 4 +- scrape/target.go | 16 +- scrape/target_test.go | 8 +- storage/fanout.go | 18 +- storage/fanout_test.go | 18 +- storage/interface.go | 12 +- storage/remote/read_handler_test.go | 2 +- storage/remote/write.go | 6 +- storage/remote/write_handler.go | 20 +-- storage/remote/write_handler_test.go | 6 +- tsdb/agent/db.go | 6 +- tsdb/agent/db_test.go | 72 ++++---- tsdb/block_test.go | 10 +- tsdb/blockwriter_test.go | 4 +- tsdb/compact_test.go | 46 ++--- tsdb/db_test.go | 256 +++++++++++++-------------- tsdb/example_test.go | 4 +- tsdb/head_append.go | 30 ++-- tsdb/head_read_test.go | 2 +- tsdb/head_test.go | 230 ++++++++++++------------ tsdb/querier_bench_test.go | 8 +- tsdb/querier_test.go | 28 +-- tsdb/testutil.go | 10 +- tsdb/tsdbblockutil.go | 6 +- web/federate_test.go | 6 +- 38 files changed, 475 insertions(+), 469 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 4a70d63bf8..d59d0cca66 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -1639,7 +1639,7 @@ func (s *readyStorage) Appender(ctx context.Context) storage.Appender { type notReadyAppender struct{} -func (n notReadyAppender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (n notReadyAppender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { return 0, tsdb.ErrNotReady } @@ -1647,11 +1647,11 @@ func (n notReadyAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, return 0, tsdb.ErrNotReady } -func (n notReadyAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (n notReadyAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { return 0, tsdb.ErrNotReady } -func (n notReadyAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (n notReadyAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { return 0, tsdb.ErrNotReady } diff --git a/cmd/prometheus/main_test.go b/cmd/prometheus/main_test.go index 4bd1c71b2d..0d2acd336a 100644 --- a/cmd/prometheus/main_test.go +++ b/cmd/prometheus/main_test.go @@ -356,11 +356,11 @@ func TestTimeMetrics(t *testing.T) { )) app := db.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 1000, 1) + _, err = app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 1000, 1, nil) require.NoError(t, err) - _, err = app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 2000, 1) + _, err = app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 2000, 1, nil) require.NoError(t, err) - _, err = app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 3000, 1) + _, err = app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 3000, 1, nil) require.NoError(t, err) require.NoError(t, app.Commit()) diff --git a/cmd/promtool/backfill.go b/cmd/promtool/backfill.go index 1408975df9..eb21a4b08e 100644 --- a/cmd/promtool/backfill.go +++ b/cmd/promtool/backfill.go @@ -171,7 +171,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn } lbls := lb.Labels() - if _, err := app.Append(0, lbls, *ts, v); err != nil { + if _, err := app.Append(0, lbls, *ts, v, nil); err != nil { return fmt.Errorf("add sample: %w", err) } diff --git a/cmd/promtool/rules.go b/cmd/promtool/rules.go index adb214b812..39b6dc8b5e 100644 --- a/cmd/promtool/rules.go +++ b/cmd/promtool/rules.go @@ -203,7 +203,7 @@ type multipleAppender struct { } func (m *multipleAppender) add(ctx context.Context, l labels.Labels, t int64, v float64) error { - if _, err := m.appender.Append(0, l, t, v); err != nil { + if _, err := m.appender.Append(0, l, t, v, nil); err != nil { return fmt.Errorf("multiappender append: %w", err) } m.currentSampleCount++ diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go index 727275aa6b..3bb66fa6dc 100644 --- a/cmd/promtool/tsdb.go +++ b/cmd/promtool/tsdb.go @@ -213,7 +213,7 @@ func (b *writeBenchmark) ingestScrapesShard(lbls []labels.Labels, scrapeCount in ref = *s.ref } - ref, err := app.Append(ref, s.labels, ts, float64(s.value)) + ref, err := app.Append(ref, s.labels, ts, float64(s.value), nil) if err != nil { panic(err) } diff --git a/promql/bench_test.go b/promql/bench_test.go index cd6d1190ca..12a50ae712 100644 --- a/promql/bench_test.go +++ b/promql/bench_test.go @@ -72,13 +72,13 @@ func setupRangeQueryTestData(stor *teststorage.TestStorage, _ *promql.Engine, in a := stor.Appender(context.Background()) ts := int64(s * interval) for i, metric := range metrics { - ref, _ := a.Append(refs[i], metric, ts, float64(s)+float64(i)/float64(len(metrics))) + ref, _ := a.Append(refs[i], metric, ts, float64(s)+float64(i)/float64(len(metrics)), nil) refs[i] = ref } // Generating a sparse time series: each label value of "l" will contain data only for // pointsPerSparseSeries points metric := labels.FromStrings("__name__", "sparse", "l", strconv.Itoa(s/pointsPerSparseSeries)) - _, err := a.Append(0, metric, ts, float64(s)/float64(len(metrics))) + _, err := a.Append(0, metric, ts, float64(s)/float64(len(metrics)), nil) if err != nil { return err } @@ -402,11 +402,11 @@ func generateNativeHistogramSeries(app storage.Appender, numSeries int) error { ts := time.Unix(int64(i*15), 0).UnixMilli() if i == 0 { // Inject a histogram with a higher schema. - if _, err := app.AppendHistogram(0, seriesLabels, ts, higherSchemaHist, nil); err != nil { + if _, err := app.AppendHistogram(0, seriesLabels, ts, higherSchemaHist, nil, nil); err != nil { return err } } - if _, err := app.AppendHistogram(0, seriesLabels, ts, histograms[i], nil); err != nil { + if _, err := app.AppendHistogram(0, seriesLabels, ts, histograms[i], nil, nil); err != nil { return err } } diff --git a/promql/engine_test.go b/promql/engine_test.go index 7c398029f5..12fcc11311 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -1536,15 +1536,15 @@ load 1ms // Add some samples with negative timestamp. db := storage.DB app := db.Appender(context.Background()) - ref, err := app.Append(0, lblsneg, -1000000, 1000) + ref, err := app.Append(0, lblsneg, -1000000, 1000, nil) require.NoError(t, err) for ts := int64(-1000000 + 1000); ts <= 0; ts += 1000 { - _, err := app.Append(ref, labels.EmptyLabels(), ts, -float64(ts/1000)+1) + _, err := app.Append(ref, labels.EmptyLabels(), ts, -float64(ts/1000)+1, nil) require.NoError(t, err) } // To test the fix for https://github.com/prometheus/prometheus/issues/8433. - _, err = app.Append(0, labels.FromStrings("__name__", "metric_timestamp"), 3600*1000, 1000) + _, err = app.Append(0, labels.FromStrings("__name__", "metric_timestamp"), 3600*1000, 1000, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -3274,16 +3274,16 @@ func TestNativeHistogram_Sum_Count_Add_AvgOperator(t *testing.T) { ts := idx0 * int64(10*time.Minute/time.Millisecond) app := storage.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("__name__", "float_series", "idx", "0"), ts, 42) + _, err := app.Append(0, labels.FromStrings("__name__", "float_series", "idx", "0"), ts, 42, nil) require.NoError(t, err) for idx1, h := range c.histograms { lbls := labels.FromStrings("__name__", seriesName, "idx", strconv.Itoa(idx1)) // Since we mutate h later, we need to create a copy here. var err error if floatHisto { - _, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat(nil)) + _, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat(nil), nil) } else { - _, err = app.AppendHistogram(0, lbls, ts, h.Copy(), nil) + _, err = app.AppendHistogram(0, lbls, ts, h.Copy(), nil, nil) } require.NoError(t, err) @@ -3291,9 +3291,9 @@ func TestNativeHistogram_Sum_Count_Add_AvgOperator(t *testing.T) { newTs := ts + int64(idx1)*int64(time.Minute/time.Millisecond) // Since we mutate h later, we need to create a copy here. if floatHisto { - _, err = app.AppendHistogram(0, lbls, newTs, nil, h.Copy().ToFloat(nil)) + _, err = app.AppendHistogram(0, lbls, newTs, nil, h.Copy().ToFloat(nil), nil) } else { - _, err = app.AppendHistogram(0, lbls, newTs, h.Copy(), nil) + _, err = app.AppendHistogram(0, lbls, newTs, h.Copy(), nil, nil) } require.NoError(t, err) } @@ -3553,9 +3553,9 @@ func TestNativeHistogram_SubOperator(t *testing.T) { // Since we mutate h later, we need to create a copy here. var err error if floatHisto { - _, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat(nil)) + _, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat(nil), nil) } else { - _, err = app.AppendHistogram(0, lbls, ts, h.Copy(), nil) + _, err = app.AppendHistogram(0, lbls, ts, h.Copy(), nil, nil) } require.NoError(t, err) } diff --git a/promql/functions_test.go b/promql/functions_test.go index 9ee0ba51dc..bee65ab0ac 100644 --- a/promql/functions_test.go +++ b/promql/functions_test.go @@ -52,7 +52,7 @@ func TestDeriv(t *testing.T) { // https://github.com/prometheus/prometheus/issues/7180 for i = 0; i < 15; i++ { jitter := 12 * i % 2 - a.Append(0, metric, start+interval*i+jitter, 1) + a.Append(0, metric, start+interval*i+jitter, 1, nil) } require.NoError(t, a.Commit()) diff --git a/promql/promqltest/test.go b/promql/promqltest/test.go index e078bcb60b..5f0aabbfdb 100644 --- a/promql/promqltest/test.go +++ b/promql/promqltest/test.go @@ -582,11 +582,11 @@ func (cmd *loadCmd) appendCustomHistogram(a storage.Appender) error { func appendSample(a storage.Appender, s promql.Sample, m labels.Labels) error { if s.H != nil { - if _, err := a.AppendHistogram(0, m, s.T, nil, s.H); err != nil { + if _, err := a.AppendHistogram(0, m, s.T, nil, s.H, nil); err != nil { return err } } else { - if _, err := a.Append(0, m, s.T, s.F); err != nil { + if _, err := a.Append(0, m, s.T, s.F, nil); err != nil { return err } } diff --git a/rules/group.go b/rules/group.go index e9ef2be3ad..6695642a2a 100644 --- a/rules/group.go +++ b/rules/group.go @@ -562,9 +562,9 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { for _, s := range vector { if s.H != nil { - _, err = app.AppendHistogram(0, s.Metric, s.T, nil, s.H) + _, err = app.AppendHistogram(0, s.Metric, s.T, nil, s.H, nil) } else { - _, err = app.Append(0, s.Metric, s.T, s.F) + _, err = app.Append(0, s.Metric, s.T, s.F, nil) } if err != nil { @@ -606,7 +606,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { for metric, lset := range g.seriesInPreviousEval[i] { if _, ok := seriesReturned[metric]; !ok { // Series no longer exposed, mark it stale. - _, err = app.Append(0, lset, timestamp.FromTime(ts.Add(-ruleQueryOffset)), math.Float64frombits(value.StaleNaN)) + _, err = app.Append(0, lset, timestamp.FromTime(ts.Add(-ruleQueryOffset)), math.Float64frombits(value.StaleNaN), nil) unwrappedErr := errors.Unwrap(err) if unwrappedErr == nil { unwrappedErr = err @@ -663,7 +663,7 @@ func (g *Group) cleanupStaleSeries(ctx context.Context, ts time.Time) { queryOffset := g.QueryOffset() for _, s := range g.staleSeries { // Rule that produced series no longer configured, mark it stale. - _, err := app.Append(0, s, timestamp.FromTime(ts.Add(-queryOffset)), math.Float64frombits(value.StaleNaN)) + _, err := app.Append(0, s, timestamp.FromTime(ts.Add(-queryOffset)), math.Float64frombits(value.StaleNaN), nil) unwrappedErr := errors.Unwrap(err) if unwrappedErr == nil { unwrappedErr = err diff --git a/rules/manager_test.go b/rules/manager_test.go index 198d6bd077..782f4786a2 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -564,9 +564,9 @@ func TestStaleness(t *testing.T) { // A time series that has two samples and then goes stale. app := st.Appender(context.Background()) - app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 0, 1) - app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 1000, 2) - app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 2000, math.Float64frombits(value.StaleNaN)) + app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 0, 1, nil) + app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 1000, 2, nil) + app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 2000, math.Float64frombits(value.StaleNaN), nil) err = app.Commit() require.NoError(t, err) @@ -942,10 +942,10 @@ func TestNotify(t *testing.T) { }) app := storage.Appender(context.Background()) - app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 1000, 2) - app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 2000, 3) - app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 5000, 3) - app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 6000, 0) + app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 1000, 2, nil) + app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 2000, 3, nil) + app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 5000, 3, nil) + app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 6000, 0, nil) err = app.Commit() require.NoError(t, err) @@ -1265,8 +1265,8 @@ func TestRuleHealthUpdates(t *testing.T) { // A time series that has two samples. app := st.Appender(context.Background()) - app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 0, 1) - app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 1000, 2) + app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 0, 1, nil) + app.Append(0, labels.FromStrings(model.MetricNameLabel, "a"), 1000, 2, nil) err = app.Commit() require.NoError(t, err) @@ -1422,7 +1422,7 @@ func TestNativeHistogramsInRecordingRules(t *testing.T) { app := db.Appender(context.Background()) for i, h := range hists { l := labels.FromStrings("__name__", "histogram_metric", "idx", strconv.Itoa(i)) - _, err := app.AppendHistogram(0, l, ts.UnixMilli(), h.Copy(), nil) + _, err := app.AppendHistogram(0, l, ts.UnixMilli(), h.Copy(), nil, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) diff --git a/scrape/helpers_test.go b/scrape/helpers_test.go index 4f7918f79e..7ab466db11 100644 --- a/scrape/helpers_test.go +++ b/scrape/helpers_test.go @@ -43,7 +43,7 @@ func (a nopAppendable) Appender(_ context.Context) storage.Appender { type nopAppender struct{} -func (a nopAppender) Append(storage.SeriesRef, labels.Labels, int64, float64) (storage.SeriesRef, error) { +func (a nopAppender) Append(storage.SeriesRef, labels.Labels, int64, float64, *storage.AppendHints) (storage.SeriesRef, error) { return 0, nil } @@ -51,11 +51,11 @@ func (a nopAppender) AppendExemplar(storage.SeriesRef, labels.Labels, exemplar.E return 0, nil } -func (a nopAppender) AppendHistogram(storage.SeriesRef, labels.Labels, int64, *histogram.Histogram, *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a nopAppender) AppendHistogram(storage.SeriesRef, labels.Labels, int64, *histogram.Histogram, *histogram.FloatHistogram, *storage.AppendHints) (storage.SeriesRef, error) { return 0, nil } -func (a nopAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a nopAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { return 0, nil } @@ -114,7 +114,7 @@ type collectResultAppender struct { pendingMetadata []metadata.Metadata } -func (a *collectResultAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (a *collectResultAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { a.mtx.Lock() defer a.mtx.Unlock() a.pendingFloats = append(a.pendingFloats, floatSample{ @@ -130,7 +130,7 @@ func (a *collectResultAppender) Append(ref storage.SeriesRef, lset labels.Labels return ref, nil } - ref, err := a.next.Append(ref, lset, t, v) + ref, err := a.next.Append(ref, lset, t, v, nil) if err != nil { return 0, err } @@ -148,7 +148,7 @@ func (a *collectResultAppender) AppendExemplar(ref storage.SeriesRef, l labels.L return a.next.AppendExemplar(ref, l, e) } -func (a *collectResultAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a *collectResultAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { a.mtx.Lock() defer a.mtx.Unlock() a.pendingHistograms = append(a.pendingHistograms, histogramSample{h: h, fh: fh, t: t, metric: l}) @@ -156,14 +156,14 @@ func (a *collectResultAppender) AppendHistogram(ref storage.SeriesRef, l labels. return 0, nil } - return a.next.AppendHistogram(ref, l, t, h, fh) + return a.next.AppendHistogram(ref, l, t, h, fh, nil) } -func (a *collectResultAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a *collectResultAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if h != nil { - return a.AppendHistogram(ref, l, ct, &histogram.Histogram{}, nil) + return a.AppendHistogram(ref, l, ct, &histogram.Histogram{}, nil, nil) } - return a.AppendHistogram(ref, l, ct, nil, &histogram.FloatHistogram{}) + return a.AppendHistogram(ref, l, ct, nil, &histogram.FloatHistogram{}, nil) } func (a *collectResultAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) { @@ -181,7 +181,7 @@ func (a *collectResultAppender) UpdateMetadata(ref storage.SeriesRef, l labels.L } func (a *collectResultAppender) AppendCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64) (storage.SeriesRef, error) { - return a.Append(ref, l, ct, 0.0) + return a.Append(ref, l, ct, 0.0, nil) } func (a *collectResultAppender) Commit() error { diff --git a/scrape/scrape.go b/scrape/scrape.go index f5f02d245f..eacedd4a0d 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -1730,9 +1730,9 @@ loop: if ctMs := p.CreatedTimestamp(); ctMs != nil { if isHistogram && sl.enableNativeHistogramIngestion { if h != nil { - ref, err = app.AppendHistogramCTZeroSample(ref, lset, t, *ctMs, h, nil) + ref, err = app.AppendHistogramCTZeroSample(ref, lset, t, *ctMs, h, nil, nil) } else { - ref, err = app.AppendHistogramCTZeroSample(ref, lset, t, *ctMs, nil, fh) + ref, err = app.AppendHistogramCTZeroSample(ref, lset, t, *ctMs, nil, fh, nil) } } else { ref, err = app.AppendCTZeroSample(ref, lset, t, *ctMs) @@ -1747,12 +1747,12 @@ loop: if isHistogram && sl.enableNativeHistogramIngestion { if h != nil { - ref, err = app.AppendHistogram(ref, lset, t, h, nil) + ref, err = app.AppendHistogram(ref, lset, t, h, nil, nil) } else { - ref, err = app.AppendHistogram(ref, lset, t, nil, fh) + ref, err = app.AppendHistogram(ref, lset, t, nil, fh, nil) } } else { - ref, err = app.Append(ref, lset, t, val) + ref, err = app.Append(ref, lset, t, val, nil) } } @@ -1864,7 +1864,7 @@ loop: if err == nil { sl.cache.forEachStale(func(lset labels.Labels) bool { // Series no longer exposed, mark it stale. - _, err = app.Append(0, lset, defTime, math.Float64frombits(value.StaleNaN)) + _, err = app.Append(0, lset, defTime, math.Float64frombits(value.StaleNaN), nil) switch { case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrDuplicateSampleForTimestamp): // Do not count these in logging, as this is expected if a target @@ -2019,7 +2019,7 @@ func (sl *scrapeLoop) addReportSample(app storage.Appender, s []byte, t int64, v lset = sl.reportSampleMutator(b.Labels()) } - ref, err := app.Append(ref, lset, t, v) + ref, err := app.Append(ref, lset, t, v, nil) switch { case err == nil: if !ok { diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index 9a70d74117..78178efc02 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -2328,7 +2328,7 @@ type errorAppender struct { collectResultAppender } -func (app *errorAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (app *errorAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { switch lset.Get(model.MetricNameLabel) { case "out_of_order": return 0, storage.ErrOutOfOrderSample @@ -2337,7 +2337,7 @@ func (app *errorAppender) Append(ref storage.SeriesRef, lset labels.Labels, t in case "out_of_bounds": return 0, storage.ErrOutOfBounds default: - return app.collectResultAppender.Append(ref, lset, t, v) + return app.collectResultAppender.Append(ref, lset, t, v, nil) } } diff --git a/scrape/target.go b/scrape/target.go index 06d4737ff9..b81d97f576 100644 --- a/scrape/target.go +++ b/scrape/target.go @@ -323,14 +323,14 @@ type limitAppender struct { i int } -func (app *limitAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (app *limitAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { if !value.IsStaleNaN(v) { app.i++ if app.i > app.limit { return 0, errSampleLimit } } - ref, err := app.Appender.Append(ref, lset, t, v) + ref, err := app.Appender.Append(ref, lset, t, v, nil) if err != nil { return 0, err } @@ -343,12 +343,12 @@ type timeLimitAppender struct { maxTime int64 } -func (app *timeLimitAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (app *timeLimitAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { if t > app.maxTime { return 0, storage.ErrOutOfBounds } - ref, err := app.Appender.Append(ref, lset, t, v) + ref, err := app.Appender.Append(ref, lset, t, v, nil) if err != nil { return 0, err } @@ -362,7 +362,7 @@ type bucketLimitAppender struct { limit int } -func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if h != nil { // Return with an early error if the histogram has too many buckets and the // schema is not exponential, in which case we can't reduce the resolution. @@ -389,7 +389,7 @@ func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labe fh = fh.ReduceResolution(fh.Schema - 1) } } - ref, err := app.Appender.AppendHistogram(ref, lset, t, h, fh) + ref, err := app.Appender.AppendHistogram(ref, lset, t, h, fh, nil) if err != nil { return 0, err } @@ -402,7 +402,7 @@ type maxSchemaAppender struct { maxSchema int32 } -func (app *maxSchemaAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (app *maxSchemaAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if h != nil { if histogram.IsExponentialSchema(h.Schema) && h.Schema > app.maxSchema { h = h.ReduceResolution(app.maxSchema) @@ -413,7 +413,7 @@ func (app *maxSchemaAppender) AppendHistogram(ref storage.SeriesRef, lset labels fh = fh.ReduceResolution(app.maxSchema) } } - ref, err := app.Appender.AppendHistogram(ref, lset, t, h, fh) + ref, err := app.Appender.AppendHistogram(ref, lset, t, h, fh, nil) if err != nil { return 0, err } diff --git a/scrape/target_test.go b/scrape/target_test.go index bd27952874..2f130629a8 100644 --- a/scrape/target_test.go +++ b/scrape/target_test.go @@ -543,7 +543,7 @@ func TestBucketLimitAppender(t *testing.T) { var err error if floatHisto { fh := c.h.Copy().ToFloat(nil) - _, err = app.AppendHistogram(0, lbls, ts, nil, fh) + _, err = app.AppendHistogram(0, lbls, ts, nil, fh, nil) if c.expectError { require.Error(t, err) } else { @@ -553,7 +553,7 @@ func TestBucketLimitAppender(t *testing.T) { } } else { h := c.h.Copy() - _, err = app.AppendHistogram(0, lbls, ts, h, nil) + _, err = app.AppendHistogram(0, lbls, ts, h, nil, nil) if c.expectError { require.Error(t, err) } else { @@ -629,12 +629,12 @@ func TestMaxSchemaAppender(t *testing.T) { var err error if floatHisto { fh := c.h.Copy().ToFloat(nil) - _, err = app.AppendHistogram(0, lbls, ts, nil, fh) + _, err = app.AppendHistogram(0, lbls, ts, nil, fh, nil) require.Equal(t, c.expectSchema, fh.Schema) require.NoError(t, err) } else { h := c.h.Copy() - _, err = app.AppendHistogram(0, lbls, ts, h, nil) + _, err = app.AppendHistogram(0, lbls, ts, h, nil, nil) require.Equal(t, c.expectSchema, h.Schema) require.NoError(t, err) } diff --git a/storage/fanout.go b/storage/fanout.go index 6ff5178955..1dff2d6852 100644 --- a/storage/fanout.go +++ b/storage/fanout.go @@ -147,14 +147,14 @@ type fanoutAppender struct { secondaries []Appender } -func (f *fanoutAppender) Append(ref SeriesRef, l labels.Labels, t int64, v float64) (SeriesRef, error) { - ref, err := f.primary.Append(ref, l, t, v) +func (f *fanoutAppender) Append(ref SeriesRef, l labels.Labels, t int64, v float64, hints *AppendHints) (SeriesRef, error) { + ref, err := f.primary.Append(ref, l, t, v, nil) if err != nil { return ref, err } for _, appender := range f.secondaries { - if _, err := appender.Append(ref, l, t, v); err != nil { + if _, err := appender.Append(ref, l, t, v, nil); err != nil { return 0, err } } @@ -175,28 +175,28 @@ func (f *fanoutAppender) AppendExemplar(ref SeriesRef, l labels.Labels, e exempl return ref, nil } -func (f *fanoutAppender) AppendHistogram(ref SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (SeriesRef, error) { - ref, err := f.primary.AppendHistogram(ref, l, t, h, fh) +func (f *fanoutAppender) AppendHistogram(ref SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *AppendHints) (SeriesRef, error) { + ref, err := f.primary.AppendHistogram(ref, l, t, h, fh, nil) if err != nil { return ref, err } for _, appender := range f.secondaries { - if _, err := appender.AppendHistogram(ref, l, t, h, fh); err != nil { + if _, err := appender.AppendHistogram(ref, l, t, h, fh, nil); err != nil { return 0, err } } return ref, nil } -func (f *fanoutAppender) AppendHistogramCTZeroSample(ref SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (SeriesRef, error) { - ref, err := f.primary.AppendHistogramCTZeroSample(ref, l, t, ct, h, fh) +func (f *fanoutAppender) AppendHistogramCTZeroSample(ref SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *AppendHints) (SeriesRef, error) { + ref, err := f.primary.AppendHistogramCTZeroSample(ref, l, t, ct, h, fh, nil) if err != nil { return ref, err } for _, appender := range f.secondaries { - if _, err := appender.AppendHistogramCTZeroSample(ref, l, t, ct, h, fh); err != nil { + if _, err := appender.AppendHistogramCTZeroSample(ref, l, t, ct, h, fh, nil); err != nil { return 0, err } } diff --git a/storage/fanout_test.go b/storage/fanout_test.go index 3eef9e3cd0..4ac4391963 100644 --- a/storage/fanout_test.go +++ b/storage/fanout_test.go @@ -38,11 +38,11 @@ func TestFanout_SelectSorted(t *testing.T) { priStorage := teststorage.New(t) defer priStorage.Close() app1 := priStorage.Appender(ctx) - app1.Append(0, inputLabel, 0, 0) + app1.Append(0, inputLabel, 0, 0, nil) inputTotalSize++ - app1.Append(0, inputLabel, 1000, 1) + app1.Append(0, inputLabel, 1000, 1, nil) inputTotalSize++ - app1.Append(0, inputLabel, 2000, 2) + app1.Append(0, inputLabel, 2000, 2, nil) inputTotalSize++ err := app1.Commit() require.NoError(t, err) @@ -50,11 +50,11 @@ func TestFanout_SelectSorted(t *testing.T) { remoteStorage1 := teststorage.New(t) defer remoteStorage1.Close() app2 := remoteStorage1.Appender(ctx) - app2.Append(0, inputLabel, 3000, 3) + app2.Append(0, inputLabel, 3000, 3, nil) inputTotalSize++ - app2.Append(0, inputLabel, 4000, 4) + app2.Append(0, inputLabel, 4000, 4, nil) inputTotalSize++ - app2.Append(0, inputLabel, 5000, 5) + app2.Append(0, inputLabel, 5000, 5, nil) inputTotalSize++ err = app2.Commit() require.NoError(t, err) @@ -63,11 +63,11 @@ func TestFanout_SelectSorted(t *testing.T) { defer remoteStorage2.Close() app3 := remoteStorage2.Appender(ctx) - app3.Append(0, inputLabel, 6000, 6) + app3.Append(0, inputLabel, 6000, 6, nil) inputTotalSize++ - app3.Append(0, inputLabel, 7000, 7) + app3.Append(0, inputLabel, 7000, 7, nil) inputTotalSize++ - app3.Append(0, inputLabel, 8000, 8) + app3.Append(0, inputLabel, 8000, 8, nil) inputTotalSize++ err = app3.Commit() diff --git a/storage/interface.go b/storage/interface.go index b7ef14ce96..991b2df8d3 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -243,6 +243,12 @@ func (f QueryableFunc) Querier(mint, maxt int64) (Querier, error) { return f(mint, maxt) } +// AppendHints specifies hints passed for appender writes. +// This is used only as an option for implementation to use. +type AppendHints struct { + DiscardOutOfOrder bool +} + // Appender provides batched appends against a storage. // It must be completed with a call to Commit or Rollback and must not be reused afterwards. // @@ -259,7 +265,7 @@ type Appender interface { // to Append() at any point. Adding the sample via Append() returns a new // reference number. // If the reference is 0 it must not be used for caching. - Append(ref SeriesRef, l labels.Labels, t int64, v float64) (SeriesRef, error) + Append(ref SeriesRef, l labels.Labels, t int64, v float64, hints *AppendHints) (SeriesRef, error) // Commit submits the collected samples and purges the batch. If Commit // returns a non-nil error, it also rolls back all modifications made in @@ -317,7 +323,7 @@ type HistogramAppender interface { // For efficiency reasons, the histogram is passed as a // pointer. AppendHistogram won't mutate the histogram, but in turn // depends on the caller to not mutate it either. - AppendHistogram(ref SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (SeriesRef, error) + AppendHistogram(ref SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *AppendHints) (SeriesRef, error) // AppendHistogramCTZeroSample adds synthetic zero sample for the given ct timestamp, // which will be associated with given series, labels and the incoming // sample's t (timestamp). AppendHistogramCTZeroSample returns error if zero sample can't be @@ -331,7 +337,7 @@ type HistogramAppender interface { // to AppendHistogramCTZeroSample() at any point. // // If the reference is 0 it must not be used for caching. - AppendHistogramCTZeroSample(ref SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (SeriesRef, error) + AppendHistogramCTZeroSample(ref SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *AppendHints) (SeriesRef, error) } // MetadataUpdater provides an interface for associating metadata to stored series. diff --git a/storage/remote/read_handler_test.go b/storage/remote/read_handler_test.go index fd7f3ad48d..1d3d8f431a 100644 --- a/storage/remote/read_handler_test.go +++ b/storage/remote/read_handler_test.go @@ -436,7 +436,7 @@ func addNativeHistogramsToTestSuite(t *testing.T, storage *teststorage.TestStora app := storage.Appender(context.TODO()) for i, fh := range tsdbutil.GenerateTestFloatHistograms(n) { - _, err := app.AppendHistogram(0, lbls, int64(i)*int64(60*time.Second/time.Millisecond), nil, fh) + _, err := app.AppendHistogram(0, lbls, int64(i)*int64(60*time.Second/time.Millisecond), nil, fh, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) diff --git a/storage/remote/write.go b/storage/remote/write.go index 20e4ed10d1..f0e688fbea 100644 --- a/storage/remote/write.go +++ b/storage/remote/write.go @@ -286,7 +286,7 @@ type timestampTracker struct { } // Append implements storage.Appender. -func (t *timestampTracker) Append(_ storage.SeriesRef, _ labels.Labels, ts int64, _ float64) (storage.SeriesRef, error) { +func (t *timestampTracker) Append(ref storage.SeriesRef, l labels.Labels, ts int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { t.samples++ if ts > t.highestTimestamp { t.highestTimestamp = ts @@ -299,7 +299,7 @@ func (t *timestampTracker) AppendExemplar(_ storage.SeriesRef, _ labels.Labels, return 0, nil } -func (t *timestampTracker) AppendHistogram(_ storage.SeriesRef, _ labels.Labels, ts int64, _ *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (t *timestampTracker) AppendHistogram(ref storage.SeriesRef, l labels.Labels, ts int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { t.histograms++ if ts > t.highestTimestamp { t.highestTimestamp = ts @@ -307,7 +307,7 @@ func (t *timestampTracker) AppendHistogram(_ storage.SeriesRef, _ labels.Labels, return 0, nil } -func (t *timestampTracker) AppendHistogramCTZeroSample(_ storage.SeriesRef, _ labels.Labels, _, _ int64, _ *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (t *timestampTracker) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, ts, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { // TODO: Implement return 0, nil } diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index 466673c99d..22baae4d0a 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -287,7 +287,7 @@ func (h *writeHandler) appendV1Samples(app storage.Appender, ss []prompb.Sample, var ref storage.SeriesRef var err error for _, s := range ss { - ref, err = app.Append(ref, labels, s.GetTimestamp(), s.GetValue()) + ref, err = app.Append(ref, labels, s.GetTimestamp(), s.GetValue(), nil) if err != nil { if errors.Is(err, storage.ErrOutOfOrderSample) || errors.Is(err, storage.ErrOutOfBounds) || @@ -304,9 +304,9 @@ func (h *writeHandler) appendV1Histograms(app storage.Appender, hh []prompb.Hist var err error for _, hp := range hh { if hp.IsFloatHistogram() { - _, err = app.AppendHistogram(0, labels, hp.Timestamp, nil, hp.ToFloatHistogram()) + _, err = app.AppendHistogram(0, labels, hp.Timestamp, nil, hp.ToFloatHistogram(), nil) } else { - _, err = app.AppendHistogram(0, labels, hp.Timestamp, hp.ToIntHistogram(), nil) + _, err = app.AppendHistogram(0, labels, hp.Timestamp, hp.ToIntHistogram(), nil, nil) } if err != nil { // Although AppendHistogram does not currently return ErrDuplicateSampleForTimestamp there is @@ -395,7 +395,7 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs * // Samples. for _, s := range ts.Samples { - ref, err = app.Append(ref, ls, s.GetTimestamp(), s.GetValue()) + ref, err = app.Append(ref, ls, s.GetTimestamp(), s.GetValue(), nil) if err == nil { rs.Samples++ continue @@ -416,9 +416,9 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs * // Native Histograms. for _, hp := range ts.Histograms { if hp.IsFloatHistogram() { - ref, err = app.AppendHistogram(ref, ls, hp.Timestamp, nil, hp.ToFloatHistogram()) + ref, err = app.AppendHistogram(ref, ls, hp.Timestamp, nil, hp.ToFloatHistogram(), nil) } else { - ref, err = app.AppendHistogram(ref, ls, hp.Timestamp, hp.ToIntHistogram(), nil) + ref, err = app.AppendHistogram(ref, ls, hp.Timestamp, hp.ToIntHistogram(), nil, nil) } if err == nil { rs.Histograms++ @@ -548,24 +548,24 @@ type timeLimitAppender struct { maxTime int64 } -func (app *timeLimitAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (app *timeLimitAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { if t > app.maxTime { return 0, fmt.Errorf("%w: timestamp is too far in the future", storage.ErrOutOfBounds) } - ref, err := app.Appender.Append(ref, lset, t, v) + ref, err := app.Appender.Append(ref, lset, t, v, nil) if err != nil { return 0, err } return ref, nil } -func (app *timeLimitAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (app *timeLimitAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if t > app.maxTime { return 0, fmt.Errorf("%w: timestamp is too far in the future", storage.ErrOutOfBounds) } - ref, err := app.Appender.AppendHistogram(ref, l, t, h, fh) + ref, err := app.Appender.AppendHistogram(ref, l, t, h, fh, nil) if err != nil { return 0, err } diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index d91949131b..812601fc28 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -833,7 +833,7 @@ func (m *mockAppendable) Appender(_ context.Context) storage.Appender { return m } -func (m *mockAppendable) Append(_ storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (m *mockAppendable) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { if m.appendSampleErr != nil { return 0, m.appendSampleErr } @@ -891,7 +891,7 @@ func (m *mockAppendable) AppendExemplar(_ storage.SeriesRef, l labels.Labels, e return 0, nil } -func (m *mockAppendable) AppendHistogram(_ storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (m *mockAppendable) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if m.appendHistogramErr != nil { return 0, m.appendHistogramErr } @@ -916,7 +916,7 @@ func (m *mockAppendable) AppendHistogram(_ storage.SeriesRef, l labels.Labels, t return 0, nil } -func (m *mockAppendable) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (m *mockAppendable) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { // AppendCTZeroSample is no-op for remote-write for now. // TODO(bwplotka/arthursens): Add support for PRW 2.0 for CT zero feature (but also we might // replace this with in-metadata CT storage, see https://github.com/prometheus/prometheus/issues/14218). diff --git a/tsdb/agent/db.go b/tsdb/agent/db.go index b2c40b2017..0545db78f7 100644 --- a/tsdb/agent/db.go +++ b/tsdb/agent/db.go @@ -783,7 +783,7 @@ type appender struct { floatHistogramSeries []*memSeries } -func (a *appender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (a *appender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { // series references and chunk references are identical for agent mode. headRef := chunks.HeadSeriesRef(ref) @@ -899,7 +899,7 @@ func (a *appender) AppendExemplar(ref storage.SeriesRef, _ labels.Labels, e exem return storage.SeriesRef(s.ref), nil } -func (a *appender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a *appender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if h != nil { if err := h.Validate(); err != nil { return 0, err @@ -971,7 +971,7 @@ func (a *appender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int return storage.SeriesRef(series.ref), nil } -func (a *appender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a *appender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { // TODO(bwplotka/arthursens): Wire metadata in the Agent's appender. return 0, nil } diff --git a/tsdb/agent/db_test.go b/tsdb/agent/db_test.go index 4d5fda25db..cdf2dd0435 100644 --- a/tsdb/agent/db_test.go +++ b/tsdb/agent/db_test.go @@ -47,23 +47,23 @@ func TestDB_InvalidSeries(t *testing.T) { app := s.Appender(context.Background()) t.Run("Samples", func(t *testing.T) { - _, err := app.Append(0, labels.Labels{}, 0, 0) + _, err := app.Append(0, labels.Labels{}, 0, 0, nil) require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject empty labels") - _, err = app.Append(0, labels.FromStrings("a", "1", "a", "2"), 0, 0) + _, err = app.Append(0, labels.FromStrings("a", "1", "a", "2"), 0, 0, nil) require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject duplicate labels") }) t.Run("Histograms", func(t *testing.T) { - _, err := app.AppendHistogram(0, labels.Labels{}, 0, tsdbutil.GenerateTestHistograms(1)[0], nil) + _, err := app.AppendHistogram(0, labels.Labels{}, 0, tsdbutil.GenerateTestHistograms(1)[0], nil, nil) require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject empty labels") - _, err = app.AppendHistogram(0, labels.FromStrings("a", "1", "a", "2"), 0, tsdbutil.GenerateTestHistograms(1)[0], nil) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "1", "a", "2"), 0, tsdbutil.GenerateTestHistograms(1)[0], nil, nil) require.ErrorIs(t, err, tsdb.ErrInvalidSample, "should reject duplicate labels") }) t.Run("Exemplars", func(t *testing.T) { - sRef, err := app.Append(0, labels.FromStrings("a", "1"), 0, 0) + sRef, err := app.Append(0, labels.FromStrings("a", "1"), 0, 0, nil) require.NoError(t, err, "should not reject valid series") _, err = app.AppendExemplar(0, labels.EmptyLabels(), exemplar.Exemplar{}) @@ -134,7 +134,7 @@ func TestCommit(t *testing.T) { for i := 0; i < numDatapoints; i++ { sample := chunks.GenerateSamples(0, 1) - ref, err := app.Append(0, lset, sample[0].T(), sample[0].F()) + ref, err := app.Append(0, lset, sample[0].T(), sample[0].F(), nil) require.NoError(t, err) e := exemplar.Exemplar{ @@ -155,7 +155,7 @@ func TestCommit(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := 0; i < numHistograms; i++ { - _, err := app.AppendHistogram(0, lset, int64(i), histograms[i], nil) + _, err := app.AppendHistogram(0, lset, int64(i), histograms[i], nil, nil) require.NoError(t, err) } } @@ -167,7 +167,7 @@ func TestCommit(t *testing.T) { floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := 0; i < numHistograms; i++ { - _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i]) + _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i], nil) require.NoError(t, err) } } @@ -249,7 +249,7 @@ func TestRollback(t *testing.T) { for i := 0; i < numDatapoints; i++ { sample := chunks.GenerateSamples(0, 1) - _, err := app.Append(0, lset, sample[0].T(), sample[0].F()) + _, err := app.Append(0, lset, sample[0].T(), sample[0].F(), nil) require.NoError(t, err) } } @@ -261,7 +261,7 @@ func TestRollback(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := 0; i < numHistograms; i++ { - _, err := app.AppendHistogram(0, lset, int64(i), histograms[i], nil) + _, err := app.AppendHistogram(0, lset, int64(i), histograms[i], nil, nil) require.NoError(t, err) } } @@ -273,7 +273,7 @@ func TestRollback(t *testing.T) { floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := 0; i < numHistograms; i++ { - _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i]) + _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i], nil) require.NoError(t, err) } } @@ -365,7 +365,7 @@ func TestFullTruncateWAL(t *testing.T) { lset := labels.New(l...) for i := 0; i < numDatapoints; i++ { - _, err := app.Append(0, lset, int64(lastTs), 0) + _, err := app.Append(0, lset, int64(lastTs), 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -378,7 +378,7 @@ func TestFullTruncateWAL(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := 0; i < numHistograms; i++ { - _, err := app.AppendHistogram(0, lset, int64(lastTs), histograms[i], nil) + _, err := app.AppendHistogram(0, lset, int64(lastTs), histograms[i], nil, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -391,7 +391,7 @@ func TestFullTruncateWAL(t *testing.T) { floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := 0; i < numHistograms; i++ { - _, err := app.AppendHistogram(0, lset, int64(lastTs), nil, floatHistograms[i]) + _, err := app.AppendHistogram(0, lset, int64(lastTs), nil, floatHistograms[i], nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -427,7 +427,7 @@ func TestPartialTruncateWAL(t *testing.T) { lset := labels.New(l...) for i := 0; i < numDatapoints; i++ { - _, err := app.Append(0, lset, lastTs, 0) + _, err := app.Append(0, lset, lastTs, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -440,7 +440,7 @@ func TestPartialTruncateWAL(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(numDatapoints) for i := 0; i < numDatapoints; i++ { - _, err := app.AppendHistogram(0, lset, lastTs, histograms[i], nil) + _, err := app.AppendHistogram(0, lset, lastTs, histograms[i], nil, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -453,7 +453,7 @@ func TestPartialTruncateWAL(t *testing.T) { floatHistograms := tsdbutil.GenerateTestFloatHistograms(numDatapoints) for i := 0; i < numDatapoints; i++ { - _, err := app.AppendHistogram(0, lset, lastTs, nil, floatHistograms[i]) + _, err := app.AppendHistogram(0, lset, lastTs, nil, floatHistograms[i], nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -466,7 +466,7 @@ func TestPartialTruncateWAL(t *testing.T) { lset := labels.New(l...) for i := 0; i < numDatapoints; i++ { - _, err := app.Append(0, lset, lastTs, 0) + _, err := app.Append(0, lset, lastTs, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -479,7 +479,7 @@ func TestPartialTruncateWAL(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(numDatapoints) for i := 0; i < numDatapoints; i++ { - _, err := app.AppendHistogram(0, lset, lastTs, histograms[i], nil) + _, err := app.AppendHistogram(0, lset, lastTs, histograms[i], nil, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -492,7 +492,7 @@ func TestPartialTruncateWAL(t *testing.T) { floatHistograms := tsdbutil.GenerateTestFloatHistograms(numDatapoints) for i := 0; i < numDatapoints; i++ { - _, err := app.AppendHistogram(0, lset, lastTs, nil, floatHistograms[i]) + _, err := app.AppendHistogram(0, lset, lastTs, nil, floatHistograms[i], nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -521,7 +521,7 @@ func TestWALReplay(t *testing.T) { lset := labels.New(l...) for i := 0; i < numDatapoints; i++ { - _, err := app.Append(0, lset, lastTs, 0) + _, err := app.Append(0, lset, lastTs, 0, nil) require.NoError(t, err) } } @@ -533,7 +533,7 @@ func TestWALReplay(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := 0; i < numHistograms; i++ { - _, err := app.AppendHistogram(0, lset, lastTs, histograms[i], nil) + _, err := app.AppendHistogram(0, lset, lastTs, histograms[i], nil, nil) require.NoError(t, err) } } @@ -545,7 +545,7 @@ func TestWALReplay(t *testing.T) { floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := 0; i < numHistograms; i++ { - _, err := app.AppendHistogram(0, lset, lastTs, nil, floatHistograms[i]) + _, err := app.AppendHistogram(0, lset, lastTs, nil, floatHistograms[i], nil) require.NoError(t, err) } } @@ -618,7 +618,7 @@ func Test_ExistingWAL_NextRef(t *testing.T) { app := db.Appender(context.Background()) for i := 0; i < seriesCount; i++ { lset := labels.FromStrings(model.MetricNameLabel, fmt.Sprintf("series_%d", i)) - _, err := app.Append(0, lset, 0, 100) + _, err := app.Append(0, lset, 0, 100, nil) require.NoError(t, err) } @@ -627,7 +627,7 @@ func Test_ExistingWAL_NextRef(t *testing.T) { // Append series for i := 0; i < histogramCount; i++ { lset := labels.FromStrings(model.MetricNameLabel, fmt.Sprintf("histogram_%d", i)) - _, err := app.AppendHistogram(0, lset, 0, histograms[i], nil) + _, err := app.AppendHistogram(0, lset, 0, histograms[i], nil, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -707,7 +707,7 @@ func TestStorage_DuplicateExemplarsIgnored(t *testing.T) { app := s.Appender(context.Background()) defer s.Close() - sRef, err := app.Append(0, labels.FromStrings("a", "1"), 0, 0) + sRef, err := app.Append(0, labels.FromStrings("a", "1"), 0, 0, nil) require.NoError(t, err, "should not reject valid series") // Write a few exemplars to our appender and call Commit(). @@ -774,7 +774,7 @@ func TestDBAllowOOOSamples(t *testing.T) { lset := labels.New(l...) for i := offset; i < numDatapoints+offset; i++ { - ref, err := app.Append(0, lset, int64(i), float64(i)) + ref, err := app.Append(0, lset, int64(i), float64(i), nil) require.NoError(t, err) e := exemplar.Exemplar{ @@ -795,7 +795,7 @@ func TestDBAllowOOOSamples(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := offset; i < numDatapoints+offset; i++ { - _, err := app.AppendHistogram(0, lset, int64(i), histograms[i-offset], nil) + _, err := app.AppendHistogram(0, lset, int64(i), histograms[i-offset], nil, nil) require.NoError(t, err) } } @@ -807,7 +807,7 @@ func TestDBAllowOOOSamples(t *testing.T) { floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := offset; i < numDatapoints+offset; i++ { - _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i-offset]) + _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i-offset], nil) require.NoError(t, err) } } @@ -838,7 +838,7 @@ func TestDBAllowOOOSamples(t *testing.T) { lset := labels.New(l...) for i := 0; i < numDatapoints; i++ { - ref, err := app.Append(0, lset, int64(i), float64(i)) + ref, err := app.Append(0, lset, int64(i), float64(i), nil) require.NoError(t, err) e := exemplar.Exemplar{ @@ -859,7 +859,7 @@ func TestDBAllowOOOSamples(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(numHistograms) for i := 0; i < numDatapoints; i++ { - _, err := app.AppendHistogram(0, lset, int64(i), histograms[i], nil) + _, err := app.AppendHistogram(0, lset, int64(i), histograms[i], nil, nil) require.NoError(t, err) } } @@ -871,7 +871,7 @@ func TestDBAllowOOOSamples(t *testing.T) { floatHistograms := tsdbutil.GenerateTestFloatHistograms(numHistograms) for i := 0; i < numDatapoints; i++ { - _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i]) + _, err := app.AppendHistogram(0, lset, int64(i), nil, floatHistograms[i], nil) require.NoError(t, err) } } @@ -905,20 +905,20 @@ func TestDBOutOfOrderTimeWindow(t *testing.T) { lbls := labelsForTest(t.Name()+"_histogram", 1) lset := labels.New(lbls[0]...) - _, err := app.AppendHistogram(0, lset, c.firstTs, tsdbutil.GenerateTestHistograms(1)[0], nil) + _, err := app.AppendHistogram(0, lset, c.firstTs, tsdbutil.GenerateTestHistograms(1)[0], nil, nil) require.NoError(t, err) err = app.Commit() require.NoError(t, err) - _, err = app.AppendHistogram(0, lset, c.secondTs, tsdbutil.GenerateTestHistograms(1)[0], nil) + _, err = app.AppendHistogram(0, lset, c.secondTs, tsdbutil.GenerateTestHistograms(1)[0], nil, nil) require.ErrorIs(t, err, c.expectedError) lbls = labelsForTest(t.Name(), 1) lset = labels.New(lbls[0]...) - _, err = app.Append(0, lset, c.firstTs, 0) + _, err = app.Append(0, lset, c.firstTs, 0, nil) require.NoError(t, err) err = app.Commit() require.NoError(t, err) - _, err = app.Append(0, lset, c.secondTs, 0) + _, err = app.Append(0, lset, c.secondTs, 0, nil) require.ErrorIs(t, err, c.expectedError) expectedAppendedSamples := float64(2) diff --git a/tsdb/block_test.go b/tsdb/block_test.go index 3589b42c17..7c98616dbf 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -680,13 +680,13 @@ func createHead(tb testing.TB, w *wlog.WL, series []storage.Series, chunkDir str switch typ { case chunkenc.ValFloat: t, v := it.At() - ref, err = app.Append(ref, lset, t, v) + ref, err = app.Append(ref, lset, t, v, nil) case chunkenc.ValHistogram: t, h := it.AtHistogram(nil) - ref, err = app.AppendHistogram(ref, lset, t, h, nil) + ref, err = app.AppendHistogram(ref, lset, t, h, nil, nil) case chunkenc.ValFloatHistogram: t, fh := it.AtFloatHistogram(nil) - ref, err = app.AppendHistogram(ref, lset, t, nil, fh) + ref, err = app.AppendHistogram(ref, lset, t, nil, fh, nil) default: err = fmt.Errorf("unknown sample type %s", typ.String()) } @@ -726,7 +726,7 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series os = append(os, sample{t: t, f: v}) continue } - ref, err = app.Append(ref, lset, t, v) + ref, err = app.Append(ref, lset, t, v, nil) require.NoError(tb, err) } require.NoError(tb, it.Err()) @@ -744,7 +744,7 @@ func createHeadWithOOOSamples(tb testing.TB, w *wlog.WL, series []storage.Series for i, lset := range oooSampleLabels { ref := storage.SeriesRef(0) for _, sample := range oooSamples[i] { - ref, err = app.Append(ref, lset, sample.T(), sample.F()) + ref, err = app.Append(ref, lset, sample.T(), sample.F(), nil) require.NoError(tb, err) oooSamplesAppended++ } diff --git a/tsdb/blockwriter_test.go b/tsdb/blockwriter_test.go index 4ec25df70a..971d43d6cb 100644 --- a/tsdb/blockwriter_test.go +++ b/tsdb/blockwriter_test.go @@ -36,10 +36,10 @@ func TestBlockWriter(t *testing.T) { // Add some series. app := w.Appender(ctx) ts1, v1 := int64(44), float64(7) - _, err = app.Append(0, labels.FromStrings("a", "b"), ts1, v1) + _, err = app.Append(0, labels.FromStrings("a", "b"), ts1, v1, nil) require.NoError(t, err) ts2, v2 := int64(55), float64(12) - _, err = app.Append(0, labels.FromStrings("c", "d"), ts2, v2) + _, err = app.Append(0, labels.FromStrings("c", "d"), ts2, v2, nil) require.NoError(t, err) require.NoError(t, app.Commit()) id, err := w.Flush(ctx) diff --git a/tsdb/compact_test.go b/tsdb/compact_test.go index 5123d6e624..66b403aff7 100644 --- a/tsdb/compact_test.go +++ b/tsdb/compact_test.go @@ -1190,7 +1190,7 @@ func BenchmarkCompactionFromHead(b *testing.B) { for ln := 0; ln < labelNames; ln++ { app := h.Appender(context.Background()) for lv := 0; lv < labelValues; lv++ { - app.Append(0, labels.FromStrings(strconv.Itoa(ln), fmt.Sprintf("%d%s%d", lv, postingsBenchSuffix, ln)), 0, 0) + app.Append(0, labels.FromStrings(strconv.Itoa(ln), fmt.Sprintf("%d%s%d", lv, postingsBenchSuffix, ln)), 0, 0, nil) } require.NoError(b, app.Commit()) } @@ -1223,10 +1223,10 @@ func BenchmarkCompactionFromOOOHead(b *testing.B) { app := h.Appender(context.Background()) for lv := 0; lv < labelValues; lv++ { lbls := labels.FromStrings(strconv.Itoa(ln), fmt.Sprintf("%d%s%d", lv, postingsBenchSuffix, ln)) - _, err = app.Append(0, lbls, int64(totalSamples), 0) + _, err = app.Append(0, lbls, int64(totalSamples), 0, nil) require.NoError(b, err) for ts := 0; ts < totalSamples; ts++ { - _, err = app.Append(0, lbls, int64(ts), float64(ts)) + _, err = app.Append(0, lbls, int64(ts), float64(ts), nil) require.NoError(b, err) } } @@ -1263,9 +1263,9 @@ func TestDisableAutoCompactions(t *testing.T) { db.DisableCompactions() app := db.Appender(context.Background()) for i := int64(0); i < 3; i++ { - _, err := app.Append(0, label, i*blockRange, 0) + _, err := app.Append(0, label, i*blockRange, 0, nil) require.NoError(t, err) - _, err = app.Append(0, label, i*blockRange+1000, 0) + _, err = app.Append(0, label, i*blockRange+1000, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -1379,11 +1379,11 @@ func TestDeleteCompactionBlockAfterFailedReload(t *testing.T) { // Add some data to the head that is enough to trigger a compaction. app := db.Appender(context.Background()) - _, err := app.Append(0, defaultLabel, 1, 0) + _, err := app.Append(0, defaultLabel, 1, 0, nil) require.NoError(t, err) - _, err = app.Append(0, defaultLabel, 2, 0) + _, err = app.Append(0, defaultLabel, 2, 0, nil) require.NoError(t, err) - _, err = app.Append(0, defaultLabel, 3+rangeToTriggerCompaction, 0) + _, err = app.Append(0, defaultLabel, 3+rangeToTriggerCompaction, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -1463,7 +1463,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) { for tsMinute := from; tsMinute <= to; tsMinute++ { var err error if floatTest { - _, err = app.AppendHistogram(0, lbls, minute(tsMinute), nil, h.ToFloat(nil)) + _, err = app.AppendHistogram(0, lbls, minute(tsMinute), nil, h.ToFloat(nil), nil) efh := h.ToFloat(nil) if tsMinute == from { efh.CounterResetHint = histogram.UnknownCounterReset @@ -1472,7 +1472,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) { } *exp = append(*exp, sample{t: minute(tsMinute), fh: efh}) } else { - _, err = app.AppendHistogram(0, lbls, minute(tsMinute), h, nil) + _, err = app.AppendHistogram(0, lbls, minute(tsMinute), h, nil, nil) eh := h.Copy() if tsMinute == from { eh.CounterResetHint = histogram.UnknownCounterReset @@ -1489,7 +1489,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) { t.Helper() app := head.Appender(ctx) for tsMinute := from; tsMinute <= to; tsMinute++ { - _, err := app.Append(0, lbls, minute(tsMinute), float64(tsMinute)) + _, err := app.Append(0, lbls, minute(tsMinute), float64(tsMinute), nil) require.NoError(t, err) *exp = append(*exp, sample{t: minute(tsMinute), f: float64(tsMinute)}) } @@ -1675,7 +1675,7 @@ func TestSparseHistogramSpaceSavings(t *testing.T) { ) for i := 0; i < numHistograms; i++ { ts := int64(i) * timeStep - ref, err = sparseApp.AppendHistogram(ref, ah.baseLabels, ts, ah.hists[i], nil) + ref, err = sparseApp.AppendHistogram(ref, ah.baseLabels, ts, ah.hists[i], nil, nil) require.NoError(t, err) } } @@ -1711,20 +1711,20 @@ func TestSparseHistogramSpaceSavings(t *testing.T) { numOldSeriesPerHistogram++ b := it.At() lbls := labels.NewBuilder(ah.baseLabels).Set("le", fmt.Sprintf("%.16f", b.Upper)).Labels() - refs[itIdx], err = oldApp.Append(refs[itIdx], lbls, ts, float64(b.Count)) + refs[itIdx], err = oldApp.Append(refs[itIdx], lbls, ts, float64(b.Count), nil) require.NoError(t, err) itIdx++ } baseName := ah.baseLabels.Get(labels.MetricName) // _count metric. countLbls := labels.NewBuilder(ah.baseLabels).Set(labels.MetricName, baseName+"_count").Labels() - _, err = oldApp.Append(0, countLbls, ts, float64(h.Count)) + _, err = oldApp.Append(0, countLbls, ts, float64(h.Count), nil) require.NoError(t, err) numOldSeriesPerHistogram++ // _sum metric. sumLbls := labels.NewBuilder(ah.baseLabels).Set(labels.MetricName, baseName+"_sum").Labels() - _, err = oldApp.Append(0, sumLbls, ts, h.Sum) + _, err = oldApp.Append(0, sumLbls, ts, h.Sum, nil) require.NoError(t, err) numOldSeriesPerHistogram++ } @@ -1999,11 +1999,11 @@ func TestDelayedCompaction(t *testing.T) { // The first compaction is expected to result in 1 block. db.DisableCompactions() app := db.Appender(context.Background()) - _, err := app.Append(0, label, 0, 0) + _, err := app.Append(0, label, 0, 0, nil) require.NoError(t, err) - _, err = app.Append(0, label, 11, 0) + _, err = app.Append(0, label, 11, 0, nil) require.NoError(t, err) - _, err = app.Append(0, label, 21, 0) + _, err = app.Append(0, label, 21, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -2030,9 +2030,9 @@ func TestDelayedCompaction(t *testing.T) { // This also ensures that no delay happens between consecutive compactions. db.DisableCompactions() app = db.Appender(context.Background()) - _, err = app.Append(0, label, 31, 0) + _, err = app.Append(0, label, 31, 0, nil) require.NoError(t, err) - _, err = app.Append(0, label, 41, 0) + _, err = app.Append(0, label, 41, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -2065,7 +2065,7 @@ func TestDelayedCompaction(t *testing.T) { db.DisableCompactions() app = db.Appender(context.Background()) - _, err = app.Append(0, label, 51, 0) + _, err = app.Append(0, label, 51, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -2133,9 +2133,9 @@ func TestDelayedCompactionDoesNotBlockUnrelatedOps(t *testing.T) { if c.whenCompactable { label := labels.FromStrings("foo", "bar") app := db.Appender(context.Background()) - _, err := app.Append(0, label, 301, 0) + _, err := app.Append(0, label, 301, 0, nil) require.NoError(t, err) - _, err = app.Append(0, label, 317, 0) + _, err = app.Append(0, label, 317, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) // The Head is compactable and will still be at the end. diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 3f0fc0c841..15ab41c795 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -216,7 +216,7 @@ func TestDataAvailableOnlyAfterCommit(t *testing.T) { ctx := context.Background() app := db.Appender(ctx) - _, err := app.Append(0, labels.FromStrings("foo", "bar"), 0, 0) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), 0, 0, nil) require.NoError(t, err) querier, err := db.Querier(0, 1) @@ -250,7 +250,7 @@ func TestNoPanicAfterWALCorruption(t *testing.T) { // Appending 121 samples because on the 121st a new chunk will be created. for i := 0; i < 121; i++ { app := db.Appender(ctx) - _, err := app.Append(0, labels.FromStrings("foo", "bar"), maxt, 0) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), maxt, 0, nil) expSamples = append(expSamples, sample{t: maxt, f: 0}) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -300,7 +300,7 @@ func TestDataNotAvailableAfterRollback(t *testing.T) { }() app := db.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("foo", "bar"), 0, 0) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), 0, 0, nil) require.NoError(t, err) err = app.Rollback() @@ -324,11 +324,11 @@ func TestDBAppenderAddRef(t *testing.T) { ctx := context.Background() app1 := db.Appender(ctx) - ref1, err := app1.Append(0, labels.FromStrings("a", "b"), 123, 0) + ref1, err := app1.Append(0, labels.FromStrings("a", "b"), 123, 0, nil) require.NoError(t, err) // Reference should already work before commit. - ref2, err := app1.Append(ref1, labels.EmptyLabels(), 124, 1) + ref2, err := app1.Append(ref1, labels.EmptyLabels(), 124, 1, nil) require.NoError(t, err) require.Equal(t, ref1, ref2) @@ -338,21 +338,21 @@ func TestDBAppenderAddRef(t *testing.T) { app2 := db.Appender(ctx) // first ref should already work in next transaction. - ref3, err := app2.Append(ref1, labels.EmptyLabels(), 125, 0) + ref3, err := app2.Append(ref1, labels.EmptyLabels(), 125, 0, nil) require.NoError(t, err) require.Equal(t, ref1, ref3) - ref4, err := app2.Append(ref1, labels.FromStrings("a", "b"), 133, 1) + ref4, err := app2.Append(ref1, labels.FromStrings("a", "b"), 133, 1, nil) require.NoError(t, err) require.Equal(t, ref1, ref4) // Reference must be valid to add another sample. - ref5, err := app2.Append(ref2, labels.EmptyLabels(), 143, 2) + ref5, err := app2.Append(ref2, labels.EmptyLabels(), 143, 2, nil) require.NoError(t, err) require.Equal(t, ref1, ref5) // Missing labels & invalid refs should fail. - _, err = app2.Append(9999999, labels.EmptyLabels(), 1, 1) + _, err = app2.Append(9999999, labels.EmptyLabels(), 1, 1, nil) require.ErrorIs(t, err, ErrInvalidSample) require.NoError(t, app2.Commit()) @@ -382,11 +382,11 @@ func TestAppendEmptyLabelsIgnored(t *testing.T) { ctx := context.Background() app1 := db.Appender(ctx) - ref1, err := app1.Append(0, labels.FromStrings("a", "b"), 123, 0) + ref1, err := app1.Append(0, labels.FromStrings("a", "b"), 123, 0, nil) require.NoError(t, err) // Add with empty label. - ref2, err := app1.Append(0, labels.FromStrings("a", "b", "c", ""), 124, 0) + ref2, err := app1.Append(0, labels.FromStrings("a", "b", "c", ""), 124, 0, nil) require.NoError(t, err) // Should be the same series. @@ -438,7 +438,7 @@ Outer: smpls := make([]float64, numSamples) for i := int64(0); i < numSamples; i++ { smpls[i] = rand.Float64() - app.Append(0, labels.FromStrings("a", "b"), i, smpls[i]) + app.Append(0, labels.FromStrings("a", "b"), i, smpls[i], nil) } require.NoError(t, app.Commit()) @@ -494,14 +494,14 @@ func TestAmendHistogramDatapointCausesError(t *testing.T) { ctx := context.Background() app := db.Appender(ctx) - _, err := app.Append(0, labels.FromStrings("a", "b"), 0, 0) + _, err := app.Append(0, labels.FromStrings("a", "b"), 0, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) app = db.Appender(ctx) - _, err = app.Append(0, labels.FromStrings("a", "b"), 0, 0) + _, err = app.Append(0, labels.FromStrings("a", "b"), 0, 0, nil) require.NoError(t, err) - _, err = app.Append(0, labels.FromStrings("a", "b"), 0, 1) + _, err = app.Append(0, labels.FromStrings("a", "b"), 0, 1, nil) require.ErrorIs(t, err, storage.ErrDuplicateSampleForTimestamp) require.NoError(t, app.Rollback()) @@ -520,29 +520,29 @@ func TestAmendHistogramDatapointCausesError(t *testing.T) { fh := h.ToFloat(nil) app = db.Appender(ctx) - _, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil, nil) require.NoError(t, err) require.NoError(t, app.Commit()) app = db.Appender(ctx) - _, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil, nil) require.NoError(t, err) h.Schema = 2 - _, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil, nil) require.Equal(t, storage.ErrDuplicateSampleForTimestamp, err) require.NoError(t, app.Rollback()) // Float histogram. app = db.Appender(ctx) - _, err = app.AppendHistogram(0, labels.FromStrings("a", "d"), 0, nil, fh.Copy()) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "d"), 0, nil, fh.Copy(), nil) require.NoError(t, err) require.NoError(t, app.Commit()) app = db.Appender(ctx) - _, err = app.AppendHistogram(0, labels.FromStrings("a", "d"), 0, nil, fh.Copy()) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "d"), 0, nil, fh.Copy(), nil) require.NoError(t, err) fh.Schema = 2 - _, err = app.AppendHistogram(0, labels.FromStrings("a", "d"), 0, nil, fh.Copy()) + _, err = app.AppendHistogram(0, labels.FromStrings("a", "d"), 0, nil, fh.Copy(), nil) require.Equal(t, storage.ErrDuplicateSampleForTimestamp, err) require.NoError(t, app.Rollback()) } @@ -555,12 +555,12 @@ func TestDuplicateNaNDatapointNoAmendError(t *testing.T) { ctx := context.Background() app := db.Appender(ctx) - _, err := app.Append(0, labels.FromStrings("a", "b"), 0, math.NaN()) + _, err := app.Append(0, labels.FromStrings("a", "b"), 0, math.NaN(), nil) require.NoError(t, err) require.NoError(t, app.Commit()) app = db.Appender(ctx) - _, err = app.Append(0, labels.FromStrings("a", "b"), 0, math.NaN()) + _, err = app.Append(0, labels.FromStrings("a", "b"), 0, math.NaN(), nil) require.NoError(t, err) } @@ -572,12 +572,12 @@ func TestNonDuplicateNaNDatapointsCausesAmendError(t *testing.T) { ctx := context.Background() app := db.Appender(ctx) - _, err := app.Append(0, labels.FromStrings("a", "b"), 0, math.Float64frombits(0x7ff0000000000001)) + _, err := app.Append(0, labels.FromStrings("a", "b"), 0, math.Float64frombits(0x7ff0000000000001), nil) require.NoError(t, err) require.NoError(t, app.Commit()) app = db.Appender(ctx) - _, err = app.Append(0, labels.FromStrings("a", "b"), 0, math.Float64frombits(0x7ff0000000000002)) + _, err = app.Append(0, labels.FromStrings("a", "b"), 0, math.Float64frombits(0x7ff0000000000002), nil) require.ErrorIs(t, err, storage.ErrDuplicateSampleForTimestamp) } @@ -589,7 +589,7 @@ func TestEmptyLabelsetCausesError(t *testing.T) { ctx := context.Background() app := db.Appender(ctx) - _, err := app.Append(0, labels.Labels{}, 0, 0) + _, err := app.Append(0, labels.Labels{}, 0, 0, nil) require.Error(t, err) require.Equal(t, "empty labelset: invalid sample", err.Error()) } @@ -603,9 +603,9 @@ func TestSkippingInvalidValuesInSameTxn(t *testing.T) { // Append AmendedValue. ctx := context.Background() app := db.Appender(ctx) - _, err := app.Append(0, labels.FromStrings("a", "b"), 0, 1) + _, err := app.Append(0, labels.FromStrings("a", "b"), 0, 1, nil) require.NoError(t, err) - _, err = app.Append(0, labels.FromStrings("a", "b"), 0, 2) + _, err = app.Append(0, labels.FromStrings("a", "b"), 0, 2, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -621,9 +621,9 @@ func TestSkippingInvalidValuesInSameTxn(t *testing.T) { // Append Out of Order Value. app = db.Appender(ctx) - _, err = app.Append(0, labels.FromStrings("a", "b"), 10, 3) + _, err = app.Append(0, labels.FromStrings("a", "b"), 10, 3, nil) require.NoError(t, err) - _, err = app.Append(0, labels.FromStrings("a", "b"), 7, 5) + _, err = app.Append(0, labels.FromStrings("a", "b"), 7, 5, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -645,7 +645,7 @@ func TestDB_Snapshot(t *testing.T) { app := db.Appender(ctx) mint := int64(1414141414000) for i := 0; i < 1000; i++ { - _, err := app.Append(0, labels.FromStrings("foo", "bar"), mint+int64(i), 1.0) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), mint+int64(i), 1.0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -691,7 +691,7 @@ func TestDB_Snapshot_ChunksOutsideOfCompactedRange(t *testing.T) { app := db.Appender(ctx) mint := int64(1414141414000) for i := 0; i < 1000; i++ { - _, err := app.Append(0, labels.FromStrings("foo", "bar"), mint+int64(i), 1.0) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), mint+int64(i), 1.0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -744,7 +744,7 @@ func TestDB_SnapshotWithDelete(t *testing.T) { smpls := make([]float64, numSamples) for i := int64(0); i < numSamples; i++ { smpls[i] = rand.Float64() - app.Append(0, labels.FromStrings("a", "b"), i, smpls[i]) + app.Append(0, labels.FromStrings("a", "b"), i, smpls[i], nil) } require.NoError(t, app.Commit()) @@ -892,7 +892,7 @@ func TestDB_e2e(t *testing.T) { series = append(series, sample{ts, v, nil, nil}) - _, err := app.Append(0, lset, ts, v) + _, err := app.Append(0, lset, ts, v, nil) require.NoError(t, err) ts += rand.Int63n(timeInterval) + 1 @@ -987,7 +987,7 @@ func TestWALFlushedOnDBClose(t *testing.T) { ctx := context.Background() app := db.Appender(ctx) - _, err := app.Append(0, lbls, 0, 1) + _, err := app.Append(0, lbls, 0, 1, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -1065,10 +1065,10 @@ func TestWALSegmentSizeOptions(t *testing.T) { for i := int64(0); i < 155; i++ { app := db.Appender(context.Background()) - ref, err := app.Append(0, labels.FromStrings("wal"+strconv.Itoa(int(i)), "size"), i, rand.Float64()) + ref, err := app.Append(0, labels.FromStrings("wal"+strconv.Itoa(int(i)), "size"), i, rand.Float64(), nil) require.NoError(t, err) for j := int64(1); j <= 78; j++ { - _, err := app.Append(ref, labels.EmptyLabels(), i+j, rand.Float64()) + _, err := app.Append(ref, labels.EmptyLabels(), i+j, rand.Float64(), nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -1127,7 +1127,7 @@ func testWALReplayRaceOnSamplesLoggedBeforeSeries(t *testing.T, numSamplesBefore lbls := labels.FromStrings("series_id", strconv.Itoa(seriesRef)) for ts := numSamplesBeforeSeriesCreation; ts < numSamplesBeforeSeriesCreation+numSamplesAfterSeriesCreation; ts++ { - _, err := app.Append(0, lbls, int64(ts), float64(ts)) + _, err := app.Append(0, lbls, int64(ts), float64(ts), nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -1179,7 +1179,7 @@ func TestTombstoneClean(t *testing.T) { smpls := make([]float64, numSamples) for i := int64(0); i < numSamples; i++ { smpls[i] = rand.Float64() - app.Append(0, labels.FromStrings("a", "b"), i, smpls[i]) + app.Append(0, labels.FromStrings("a", "b"), i, smpls[i], nil) } require.NoError(t, app.Commit()) @@ -1273,7 +1273,7 @@ func TestTombstoneCleanResultEmptyBlock(t *testing.T) { smpls := make([]float64, numSamples) for i := int64(0); i < numSamples; i++ { smpls[i] = rand.Float64() - app.Append(0, labels.FromStrings("a", "b"), i, smpls[i]) + app.Append(0, labels.FromStrings("a", "b"), i, smpls[i], nil) } require.NoError(t, app.Commit()) @@ -1573,7 +1573,7 @@ func TestSizeRetention(t *testing.T) { it = s.Iterator(it) for it.Next() == chunkenc.ValFloat { tim, v := it.At() - _, err := headApp.Append(0, s.Labels(), tim, v) + _, err := headApp.Append(0, s.Labels(), tim, v, nil) require.NoError(t, err) } require.NoError(t, it.Err()) @@ -1631,7 +1631,7 @@ func TestSizeRetention(t *testing.T) { // Add some out of order samples to check the size of WBL. headApp = db.Head().Appender(context.Background()) for ts := int64(750); ts < 800; ts++ { - _, err := headApp.Append(0, aSeries, ts, float64(ts)) + _, err := headApp.Append(0, aSeries, ts, float64(ts), nil) require.NoError(t, err) } require.NoError(t, headApp.Commit()) @@ -1714,7 +1714,7 @@ func TestNotMatcherSelectsLabelsUnsetSeries(t *testing.T) { ctx := context.Background() app := db.Appender(ctx) for _, lbls := range labelpairs { - _, err := app.Append(0, lbls, 0, 1) + _, err := app.Append(0, lbls, 0, 1, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -1900,9 +1900,9 @@ func TestChunkAtBlockBoundary(t *testing.T) { label := labels.FromStrings("foo", "bar") for i := int64(0); i < 3; i++ { - _, err := app.Append(0, label, i*blockRange, 0) + _, err := app.Append(0, label, i*blockRange, 0, nil) require.NoError(t, err) - _, err = app.Append(0, label, i*blockRange+1000, 0) + _, err = app.Append(0, label, i*blockRange+1000, 0, nil) require.NoError(t, err) } @@ -1956,9 +1956,9 @@ func TestQuerierWithBoundaryChunks(t *testing.T) { label := labels.FromStrings("foo", "bar") for i := int64(0); i < 5; i++ { - _, err := app.Append(0, label, i*blockRange, 0) + _, err := app.Append(0, label, i*blockRange, 0, nil) require.NoError(t, err) - _, err = app.Append(0, labels.FromStrings("blockID", strconv.FormatInt(i, 10)), i*blockRange, 0) + _, err = app.Append(0, labels.FromStrings("blockID", strconv.FormatInt(i, 10)), i*blockRange, 0, nil) require.NoError(t, err) } @@ -2003,7 +2003,7 @@ func TestInitializeHeadTimestamp(t *testing.T) { // First added sample initializes the writable range. ctx := context.Background() app := db.Appender(ctx) - _, err = app.Append(0, labels.FromStrings("a", "b"), 1000, 1) + _, err = app.Append(0, labels.FromStrings("a", "b"), 1000, 1, nil) require.NoError(t, err) require.Equal(t, int64(1000), db.head.MinTime()) @@ -2112,11 +2112,11 @@ func TestNoEmptyBlocks(t *testing.T) { t.Run("Test no blocks after deleting all samples from head.", func(t *testing.T) { app := db.Appender(ctx) - _, err := app.Append(0, defaultLabel, 1, 0) + _, err := app.Append(0, defaultLabel, 1, 0, nil) require.NoError(t, err) - _, err = app.Append(0, defaultLabel, 2, 0) + _, err = app.Append(0, defaultLabel, 2, 0, nil) require.NoError(t, err) - _, err = app.Append(0, defaultLabel, 3+rangeToTriggerCompaction, 0) + _, err = app.Append(0, defaultLabel, 3+rangeToTriggerCompaction, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.NoError(t, db.Delete(ctx, math.MinInt64, math.MaxInt64, defaultMatcher)) @@ -2129,16 +2129,16 @@ func TestNoEmptyBlocks(t *testing.T) { require.Empty(t, actBlocks) app = db.Appender(ctx) - _, err = app.Append(0, defaultLabel, 1, 0) + _, err = app.Append(0, defaultLabel, 1, 0, nil) require.Equal(t, storage.ErrOutOfBounds, err, "the head should be truncated so no samples in the past should be allowed") // Adding new blocks. currentTime := db.Head().MaxTime() - _, err = app.Append(0, defaultLabel, currentTime, 0) + _, err = app.Append(0, defaultLabel, currentTime, 0, nil) require.NoError(t, err) - _, err = app.Append(0, defaultLabel, currentTime+1, 0) + _, err = app.Append(0, defaultLabel, currentTime+1, 0, nil) require.NoError(t, err) - _, err = app.Append(0, defaultLabel, currentTime+rangeToTriggerCompaction, 0) + _, err = app.Append(0, defaultLabel, currentTime+rangeToTriggerCompaction, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -2155,11 +2155,11 @@ func TestNoEmptyBlocks(t *testing.T) { oldBlocks := db.Blocks() app := db.Appender(ctx) currentTime := db.Head().MaxTime() - _, err := app.Append(0, defaultLabel, currentTime, 0) + _, err := app.Append(0, defaultLabel, currentTime, 0, nil) require.NoError(t, err) - _, err = app.Append(0, defaultLabel, currentTime+1, 0) + _, err = app.Append(0, defaultLabel, currentTime+1, 0, nil) require.NoError(t, err) - _, err = app.Append(0, defaultLabel, currentTime+rangeToTriggerCompaction, 0) + _, err = app.Append(0, defaultLabel, currentTime+rangeToTriggerCompaction, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.NoError(t, db.head.Delete(ctx, math.MinInt64, math.MaxInt64, defaultMatcher)) @@ -2241,7 +2241,7 @@ func TestDB_LabelNames(t *testing.T) { for i := mint; i <= maxt; i++ { for _, tuple := range sampleLabels { label := labels.FromStrings(tuple[0], tuple[1]) - _, err := app.Append(0, label, i*blockRange, 0) + _, err := app.Append(0, label, i*blockRange, 0, nil) require.NoError(t, err) } } @@ -2310,7 +2310,7 @@ func TestCorrectNumTombstones(t *testing.T) { app := db.Appender(ctx) for i := int64(0); i < 3; i++ { for j := int64(0); j < 15; j++ { - _, err := app.Append(0, defaultLabel, i*blockRange+j, 0) + _, err := app.Append(0, defaultLabel, i*blockRange+j, 0, nil) require.NoError(t, err) } } @@ -2362,14 +2362,14 @@ func TestBlockRanges(t *testing.T) { app := db.Appender(ctx) lbl := labels.FromStrings("a", "b") - _, err = app.Append(0, lbl, firstBlockMaxT-1, rand.Float64()) + _, err = app.Append(0, lbl, firstBlockMaxT-1, rand.Float64(), nil) require.Error(t, err, "appending a sample with a timestamp covered by a previous block shouldn't be possible") - _, err = app.Append(0, lbl, firstBlockMaxT+1, rand.Float64()) + _, err = app.Append(0, lbl, firstBlockMaxT+1, rand.Float64(), nil) require.NoError(t, err) - _, err = app.Append(0, lbl, firstBlockMaxT+2, rand.Float64()) + _, err = app.Append(0, lbl, firstBlockMaxT+2, rand.Float64(), nil) require.NoError(t, err) secondBlockMaxt := firstBlockMaxT + rangeToTriggerCompaction - _, err = app.Append(0, lbl, secondBlockMaxt, rand.Float64()) // Add samples to trigger a new compaction + _, err = app.Append(0, lbl, secondBlockMaxt, rand.Float64(), nil) // Add samples to trigger a new compaction require.NoError(t, err) require.NoError(t, app.Commit()) @@ -2388,13 +2388,13 @@ func TestBlockRanges(t *testing.T) { // and compaction doesn't create an overlapping block. app = db.Appender(ctx) db.DisableCompactions() - _, err = app.Append(0, lbl, secondBlockMaxt+1, rand.Float64()) + _, err = app.Append(0, lbl, secondBlockMaxt+1, rand.Float64(), nil) require.NoError(t, err) - _, err = app.Append(0, lbl, secondBlockMaxt+2, rand.Float64()) + _, err = app.Append(0, lbl, secondBlockMaxt+2, rand.Float64(), nil) require.NoError(t, err) - _, err = app.Append(0, lbl, secondBlockMaxt+3, rand.Float64()) + _, err = app.Append(0, lbl, secondBlockMaxt+3, rand.Float64(), nil) require.NoError(t, err) - _, err = app.Append(0, lbl, secondBlockMaxt+4, rand.Float64()) + _, err = app.Append(0, lbl, secondBlockMaxt+4, rand.Float64(), nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.NoError(t, db.Close()) @@ -2410,7 +2410,7 @@ func TestBlockRanges(t *testing.T) { require.Equal(t, db.Blocks()[2].Meta().MaxTime, thirdBlockMaxt, "unexpected maxt of the last block") app = db.Appender(ctx) - _, err = app.Append(0, lbl, thirdBlockMaxt+rangeToTriggerCompaction, rand.Float64()) // Trigger a compaction + _, err = app.Append(0, lbl, thirdBlockMaxt+rangeToTriggerCompaction, rand.Float64(), nil) // Trigger a compaction require.NoError(t, err) require.NoError(t, app.Commit()) for x := 0; x < 100; x++ { @@ -2472,7 +2472,7 @@ func TestDBReadOnly(t *testing.T) { dbSizeBeforeAppend, err := fileutil.DirSize(dbWritable.Dir()) require.NoError(t, err) app := dbWritable.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("foo", "bar"), dbWritable.Head().MaxTime()+1, 0) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), dbWritable.Head().MaxTime()+1, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -2583,7 +2583,7 @@ func TestDBReadOnly_FlushWAL(t *testing.T) { app := db.Appender(ctx) maxt = 1000 for i := 0; i < maxt; i++ { - _, err := app.Append(0, labels.FromStrings(defaultLabelName, "flush"), int64(i), 1.0) + _, err := app.Append(0, labels.FromStrings(defaultLabelName, "flush"), int64(i), 1.0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -2668,7 +2668,7 @@ func TestDBReadOnly_Querier_NoAlteration(t *testing.T) { // Append until the first mmapped head chunk. for i := 0; i < 121; i++ { app := db.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("foo", "bar"), int64(i), 0) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), int64(i), 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) } @@ -2725,7 +2725,7 @@ func TestDBCannotSeePartialCommits(t *testing.T) { app := db.Appender(ctx) for j := 0; j < 100; j++ { - _, err := app.Append(0, labels.FromStrings("foo", "bar", "a", strconv.Itoa(j)), int64(iter), float64(iter)) + _, err := app.Append(0, labels.FromStrings("foo", "bar", "a", strconv.Itoa(j)), int64(iter), float64(iter), nil) require.NoError(t, err) } err = app.Commit() @@ -2791,7 +2791,7 @@ func TestDBQueryDoesntSeeAppendsAfterCreation(t *testing.T) { ctx := context.Background() app := db.Appender(ctx) - _, err = app.Append(0, labels.FromStrings("foo", "bar"), 0, 0) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), 0, 0, nil) require.NoError(t, err) querierAfterAddButBeforeCommit, err := db.Querier(0, 1000000) @@ -3119,7 +3119,7 @@ func TestCompactHead(t *testing.T) { maxt := 100 for i := 0; i < maxt; i++ { val := rand.Float64() - _, err := app.Append(0, labels.FromStrings("a", "b"), int64(i), val) + _, err := app.Append(0, labels.FromStrings("a", "b"), int64(i), val, nil) require.NoError(t, err) expSamples = append(expSamples, sample{int64(i), val, nil, nil}) } @@ -3165,7 +3165,7 @@ func TestCompactHeadWithDeletion(t *testing.T) { ctx := context.Background() app := db.Appender(ctx) - _, err = app.Append(0, labels.FromStrings("a", "b"), 10, rand.Float64()) + _, err = app.Append(0, labels.FromStrings("a", "b"), 10, rand.Float64(), nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -3341,9 +3341,9 @@ func TestOneCheckpointPerCompactCall(t *testing.T) { // Append samples spanning 59 block ranges. app := db.Appender(context.Background()) for i := int64(0); i < 60; i++ { - _, err := app.Append(0, lbls, blockRange*i, rand.Float64()) + _, err := app.Append(0, lbls, blockRange*i, rand.Float64(), nil) require.NoError(t, err) - _, err = app.Append(0, lbls, (blockRange*i)+blockRange/2, rand.Float64()) + _, err = app.Append(0, lbls, (blockRange*i)+blockRange/2, rand.Float64(), nil) require.NoError(t, err) // Rotate the WAL file so that there is >3 files for checkpoint to happen. _, err = db.head.wal.NextSegment() @@ -3401,7 +3401,7 @@ func TestOneCheckpointPerCompactCall(t *testing.T) { // Adding sample way into the future. app = db.Appender(context.Background()) - _, err = app.Append(0, lbls, blockRange*120, rand.Float64()) + _, err = app.Append(0, lbls, blockRange*120, rand.Float64(), nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -3507,7 +3507,7 @@ func testQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t app := db.Appender(ctx) for _, metric := range metrics { - _, err := app.Append(0, metric, ts, float64(ts)) + _, err := app.Append(0, metric, ts, float64(ts), nil) require.NoError(t, err) } @@ -3523,7 +3523,7 @@ func testQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t app := db.Appender(ctx) for _, metric := range metrics { - _, err := app.Append(0, metric, ts, float64(ts)) + _, err := app.Append(0, metric, ts, float64(ts), nil) require.NoError(t, err) } @@ -3643,7 +3643,7 @@ func testChunkQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChun app := db.Appender(ctx) for _, metric := range metrics { - _, err := app.Append(0, metric, ts, float64(ts)) + _, err := app.Append(0, metric, ts, float64(ts), nil) require.NoError(t, err) } @@ -3659,7 +3659,7 @@ func testChunkQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChun app := db.Appender(ctx) for _, metric := range metrics { - _, err := app.Append(0, metric, ts, float64(ts)) + _, err := app.Append(0, metric, ts, float64(ts), nil) require.NoError(t, err) } @@ -3746,7 +3746,7 @@ func TestQuerierShouldNotFailIfOOOCompactionOccursAfterRetrievingQuerier(t *test // Push samples after the OOO sample we'll write below. for ; ts < 10*interval; ts += interval { app := db.Appender(ctx) - _, err := app.Append(0, metric, ts, float64(ts)) + _, err := app.Append(0, metric, ts, float64(ts), nil) require.NoError(t, err) require.NoError(t, app.Commit()) samplesWritten++ @@ -3754,7 +3754,7 @@ func TestQuerierShouldNotFailIfOOOCompactionOccursAfterRetrievingQuerier(t *test // Push a single OOO sample. app := db.Appender(ctx) - _, err := app.Append(0, metric, oooTS, float64(ts)) + _, err := app.Append(0, metric, oooTS, float64(ts), nil) require.NoError(t, err) require.NoError(t, app.Commit()) samplesWritten++ @@ -3840,7 +3840,7 @@ func TestQuerierShouldNotFailIfOOOCompactionOccursAfterSelecting(t *testing.T) { // Push samples after the OOO sample we'll write below. for ; ts < 10*interval; ts += interval { app := db.Appender(ctx) - _, err := app.Append(0, metric, ts, float64(ts)) + _, err := app.Append(0, metric, ts, float64(ts), nil) require.NoError(t, err) require.NoError(t, app.Commit()) samplesWritten++ @@ -3848,7 +3848,7 @@ func TestQuerierShouldNotFailIfOOOCompactionOccursAfterSelecting(t *testing.T) { // Push a single OOO sample. app := db.Appender(ctx) - _, err := app.Append(0, metric, oooTS, float64(ts)) + _, err := app.Append(0, metric, oooTS, float64(ts), nil) require.NoError(t, err) require.NoError(t, app.Commit()) samplesWritten++ @@ -3922,7 +3922,7 @@ func TestQuerierShouldNotFailIfOOOCompactionOccursAfterRetrievingIterators(t *te // Push samples after the OOO sample we'll write below. for ; ts < 10*interval; ts += interval { app := db.Appender(ctx) - _, err := app.Append(0, metric, ts, float64(ts)) + _, err := app.Append(0, metric, ts, float64(ts), nil) require.NoError(t, err) require.NoError(t, app.Commit()) samplesWritten++ @@ -3930,7 +3930,7 @@ func TestQuerierShouldNotFailIfOOOCompactionOccursAfterRetrievingIterators(t *te // Push a single OOO sample. app := db.Appender(ctx) - _, err := app.Append(0, metric, oooTS, float64(ts)) + _, err := app.Append(0, metric, oooTS, float64(ts), nil) require.NoError(t, err) require.NoError(t, app.Commit()) samplesWritten++ @@ -4009,7 +4009,7 @@ func TestOOOWALWrite(t *testing.T) { }{ "float": { appendSample: func(app storage.Appender, l labels.Labels, mins int64) (storage.SeriesRef, error) { - seriesRef, err := app.Append(0, l, minutes(mins), float64(mins)) + seriesRef, err := app.Append(0, l, minutes(mins), float64(mins), nil) require.NoError(t, err) return seriesRef, nil }, @@ -4100,7 +4100,7 @@ func TestOOOWALWrite(t *testing.T) { }, "integer histogram": { appendSample: func(app storage.Appender, l labels.Labels, mins int64) (storage.SeriesRef, error) { - seriesRef, err := app.AppendHistogram(0, l, minutes(mins), tsdbutil.GenerateTestHistogram(int(mins)), nil) + seriesRef, err := app.AppendHistogram(0, l, minutes(mins), tsdbutil.GenerateTestHistogram(int(mins)), nil, nil) require.NoError(t, err) return seriesRef, nil }, @@ -4191,7 +4191,7 @@ func TestOOOWALWrite(t *testing.T) { }, "float histogram": { appendSample: func(app storage.Appender, l labels.Labels, mins int64) (storage.SeriesRef, error) { - seriesRef, err := app.AppendHistogram(0, l, minutes(mins), nil, tsdbutil.GenerateTestFloatHistogram(int(mins))) + seriesRef, err := app.AppendHistogram(0, l, minutes(mins), nil, tsdbutil.GenerateTestFloatHistogram(int(mins)), nil) require.NoError(t, err) return seriesRef, nil }, @@ -4416,7 +4416,7 @@ func TestDBPanicOnMmappingHeadChunk(t *testing.T) { var ref storage.SeriesRef lbls := labels.FromStrings("__name__", "testing", "foo", "bar") for i := 0; i < numSamples; i++ { - ref, err = app.Append(ref, lbls, lastTs, float64(lastTs)) + ref, err = app.Append(ref, lbls, lastTs, float64(lastTs), nil) require.NoError(t, err) lastTs += itvl if i%10 == 0 { @@ -4474,7 +4474,7 @@ func TestMetadataInWAL(t *testing.T) { s4 := labels.FromStrings("g", "h") for _, s := range []labels.Labels{s1, s2, s3, s4} { - _, err := app.Append(0, s, 0, 0) + _, err := app.Append(0, s, 0, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -4540,7 +4540,7 @@ func TestMetadataCheckpointingOnlyKeepsLatestEntry(t *testing.T) { s4 := labels.FromStrings("g", "h") for _, s := range []labels.Labels{s1, s2, s3, s4} { - _, err := app.Append(0, s, 0, 0) + _, err := app.Append(0, s, 0, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -4643,7 +4643,7 @@ func TestMetadataAssertInMemoryData(t *testing.T) { s4 := labels.FromStrings("g", "h") for _, s := range []labels.Labels{s1, s2, s3, s4} { - _, err := app.Append(0, s, 0, 0) + _, err := app.Append(0, s, 0, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -4730,18 +4730,18 @@ func TestMultipleEncodingsCommitOrder(t *testing.T) { addSample := func(app storage.Appender, ts int64, valType chunkenc.ValueType) chunks.Sample { if valType == chunkenc.ValFloat { - _, err := app.Append(0, labels.FromStrings("foo", "bar1"), ts, float64(ts)) + _, err := app.Append(0, labels.FromStrings("foo", "bar1"), ts, float64(ts), nil) require.NoError(t, err) return sample{t: ts, f: float64(ts)} } if valType == chunkenc.ValHistogram { h := tsdbutil.GenerateTestHistogram(int(ts)) - _, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil) + _, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil, nil) require.NoError(t, err) return sample{t: ts, h: h} } fh := tsdbutil.GenerateTestFloatHistogram(int(ts)) - _, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, nil, fh) + _, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, nil, fh, nil) require.NoError(t, err) return sample{t: ts, fh: fh} } @@ -5418,7 +5418,7 @@ func TestQuerierOOOQuery(t *testing.T) { }{ "float": { appendFunc: func(app storage.Appender, ts int64, counterReset bool) (storage.SeriesRef, error) { - return app.Append(0, labels.FromStrings("foo", "bar1"), ts, float64(ts)) + return app.Append(0, labels.FromStrings("foo", "bar1"), ts, float64(ts), nil) }, sampleFunc: func(ts int64) chunks.Sample { return sample{t: ts, f: float64(ts)} @@ -5430,7 +5430,7 @@ func TestQuerierOOOQuery(t *testing.T) { if counterReset { h.CounterResetHint = histogram.CounterReset } - return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil) + return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil, nil) }, sampleFunc: func(ts int64) chunks.Sample { return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(ts))} @@ -5442,7 +5442,7 @@ func TestQuerierOOOQuery(t *testing.T) { if counterReset { fh.CounterResetHint = histogram.CounterReset } - return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, nil, fh) + return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, nil, fh, nil) }, sampleFunc: func(ts int64) chunks.Sample { return sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(ts))} @@ -5453,7 +5453,7 @@ func TestQuerierOOOQuery(t *testing.T) { appendFunc: func(app storage.Appender, ts int64, counterReset bool) (storage.SeriesRef, error) { h := tsdbutil.GenerateTestHistogram(int(ts)) h.CounterResetHint = histogram.CounterReset // For this scenario, ignore the counterReset argument. - return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil) + return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil, nil) }, sampleFunc: func(ts int64) chunks.Sample { return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(ts))} @@ -5734,7 +5734,7 @@ func TestChunkQuerierOOOQuery(t *testing.T) { }{ "float": { appendFunc: func(app storage.Appender, ts int64, counterReset bool) (storage.SeriesRef, error) { - return app.Append(0, labels.FromStrings("foo", "bar1"), ts, float64(ts)) + return app.Append(0, labels.FromStrings("foo", "bar1"), ts, float64(ts), nil) }, sampleFunc: func(ts int64) chunks.Sample { return sample{t: ts, f: float64(ts)} @@ -5746,7 +5746,7 @@ func TestChunkQuerierOOOQuery(t *testing.T) { if counterReset { h.CounterResetHint = histogram.CounterReset } - return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil) + return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil, nil) }, sampleFunc: func(ts int64) chunks.Sample { return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(ts))} @@ -5758,7 +5758,7 @@ func TestChunkQuerierOOOQuery(t *testing.T) { if counterReset { fh.CounterResetHint = histogram.CounterReset } - return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, nil, fh) + return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, nil, fh, nil) }, sampleFunc: func(ts int64) chunks.Sample { return sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(ts))} @@ -5769,7 +5769,7 @@ func TestChunkQuerierOOOQuery(t *testing.T) { appendFunc: func(app storage.Appender, ts int64, counterReset bool) (storage.SeriesRef, error) { h := tsdbutil.GenerateTestHistogram(int(ts)) h.CounterResetHint = histogram.CounterReset // For this scenario, ignore the counterReset argument. - return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil) + return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil, nil) }, sampleFunc: func(ts int64) chunks.Sample { return sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(ts))} @@ -5779,7 +5779,7 @@ func TestChunkQuerierOOOQuery(t *testing.T) { // Histograms have increasing number of buckets so their chunks are recoded. appendFunc: func(app storage.Appender, ts int64, counterReset bool) (storage.SeriesRef, error) { n := ts / time.Minute.Milliseconds() - return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, nBucketHistogram(n), nil) + return app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, nBucketHistogram(n), nil, nil) }, sampleFunc: func(ts int64) chunks.Sample { n := ts / time.Minute.Milliseconds() @@ -6673,7 +6673,7 @@ func TestOOOHistogramCompactionWithCounterResets(t *testing.T) { if floatHistogram { h := tsdbutil.GenerateTestFloatHistogram(val) h.CounterResetHint = hint - _, err = app.AppendHistogram(0, l, tsMs, nil, h) + _, err = app.AppendHistogram(0, l, tsMs, nil, h, nil) require.NoError(t, err) require.NoError(t, app.Commit()) return sample{t: tsMs, fh: h.Copy()} @@ -6681,7 +6681,7 @@ func TestOOOHistogramCompactionWithCounterResets(t *testing.T) { h := tsdbutil.GenerateTestHistogram(val) h.CounterResetHint = hint - _, err = app.AppendHistogram(0, l, tsMs, h, nil) + _, err = app.AppendHistogram(0, l, tsMs, h, nil, nil) require.NoError(t, err) require.NoError(t, app.Commit()) return sample{t: tsMs, h: h.Copy()} @@ -7186,7 +7186,7 @@ func TestWBLCorruption(t *testing.T) { app := db.Appender(context.Background()) for m := fromMins; m <= toMins; m++ { ts := m * time.Minute.Milliseconds() - _, err := app.Append(0, series1, ts, float64(ts)) + _, err := app.Append(0, series1, ts, float64(ts), nil) require.NoError(t, err) allSamples = append(allSamples, sample{t: ts, f: float64(ts)}) if afterRestart { @@ -8052,12 +8052,12 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { var err error app := db.Appender(ctx) if floatHistogram { - _, err = app.AppendHistogram(0, lbls, minute(tsMinute), nil, h.ToFloat(nil)) + _, err = app.AppendHistogram(0, lbls, minute(tsMinute), nil, h.ToFloat(nil), nil) efh := h.ToFloat(nil) efh.CounterResetHint = expCRH *exp = append(*exp, sample{t: minute(tsMinute), fh: efh}) } else { - _, err = app.AppendHistogram(0, lbls, minute(tsMinute), h.Copy(), nil) + _, err = app.AppendHistogram(0, lbls, minute(tsMinute), h.Copy(), nil, nil) eh := h.Copy() eh.CounterResetHint = expCRH *exp = append(*exp, sample{t: minute(tsMinute), h: eh}) @@ -8068,7 +8068,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) { appendFloat := func(lbls labels.Labels, tsMinute int, val float64, exp *[]chunks.Sample) { t.Helper() app := db.Appender(ctx) - _, err := app.Append(0, lbls, minute(tsMinute), val) + _, err := app.Append(0, lbls, minute(tsMinute), val, nil) require.NoError(t, err) require.NoError(t, app.Commit()) *exp = append(*exp, sample{t: minute(tsMinute), f: val}) @@ -8459,22 +8459,22 @@ func TestNativeHistogramFlag(t *testing.T) { app := db.Appender(context.Background()) // Disabled by default. - _, err = app.AppendHistogram(0, l, 100, h, nil) + _, err = app.AppendHistogram(0, l, 100, h, nil, nil) require.Equal(t, storage.ErrNativeHistogramsDisabled, err) - _, err = app.AppendHistogram(0, l, 105, nil, h.ToFloat(nil)) + _, err = app.AppendHistogram(0, l, 105, nil, h.ToFloat(nil), nil) require.Equal(t, storage.ErrNativeHistogramsDisabled, err) // Enable and append. db.EnableNativeHistograms() - _, err = app.AppendHistogram(0, l, 200, h, nil) + _, err = app.AppendHistogram(0, l, 200, h, nil, nil) require.NoError(t, err) - _, err = app.AppendHistogram(0, l, 205, nil, h.ToFloat(nil)) + _, err = app.AppendHistogram(0, l, 205, nil, h.ToFloat(nil), nil) require.NoError(t, err) db.DisableNativeHistograms() - _, err = app.AppendHistogram(0, l, 300, h, nil) + _, err = app.AppendHistogram(0, l, 300, h, nil, nil) require.Equal(t, storage.ErrNativeHistogramsDisabled, err) - _, err = app.AppendHistogram(0, l, 305, nil, h.ToFloat(nil)) + _, err = app.AppendHistogram(0, l, 305, nil, h.ToFloat(nil), nil) require.Equal(t, storage.ErrNativeHistogramsDisabled, err) require.NoError(t, app.Commit()) @@ -8516,10 +8516,10 @@ func TestOOONativeHistogramFlag(t *testing.T) { db.EnableOOONativeHistograms() app := db.Appender(context.Background()) - _, err := app.AppendHistogram(0, l, 100, h, nil) + _, err := app.AppendHistogram(0, l, 100, h, nil, nil) require.NoError(t, err) - _, err = app.AppendHistogram(0, l, 50, h, nil) + _, err = app.AppendHistogram(0, l, 50, h, nil, nil) require.NoError(t, err) // The OOO sample is not detected until it is committed, so no error is returned require.NoError(t, app.Commit()) @@ -8545,11 +8545,11 @@ func TestOOONativeHistogramFlag(t *testing.T) { // Attempt to add an in-order sample app := db.Appender(context.Background()) - _, err := app.AppendHistogram(0, l, 200, h, nil) + _, err := app.AppendHistogram(0, l, 200, h, nil, nil) require.Equal(t, storage.ErrNativeHistogramsDisabled, err) // Attempt to add an OOO sample - _, err = app.AppendHistogram(0, l, 100, h, nil) + _, err = app.AppendHistogram(0, l, 100, h, nil, nil) require.Equal(t, storage.ErrNativeHistogramsDisabled, err) require.NoError(t, app.Commit()) @@ -8573,13 +8573,13 @@ func TestOOONativeHistogramFlag(t *testing.T) { // Add in-order samples app := db.Appender(context.Background()) - _, err := app.AppendHistogram(0, l, 200, h, nil) + _, err := app.AppendHistogram(0, l, 200, h, nil, nil) require.NoError(t, err) // Add OOO samples - _, err = app.AppendHistogram(0, l, 100, h, nil) + _, err = app.AppendHistogram(0, l, 100, h, nil, nil) require.NoError(t, err) - _, err = app.AppendHistogram(0, l, 150, h, nil) + _, err = app.AppendHistogram(0, l, 150, h, nil, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -8672,7 +8672,7 @@ func TestChunkQuerierReadWriteRace(t *testing.T) { app := db.Appender(context.Background()) for j := 0; j < 10; j++ { ts++ - _, err := app.Append(0, lbls, int64(ts), float64(ts*100)) + _, err := app.Append(0, lbls, int64(ts), float64(ts*100), nil) if err != nil { return err } diff --git a/tsdb/example_test.go b/tsdb/example_test.go index 46deae5198..3bce640f3c 100644 --- a/tsdb/example_test.go +++ b/tsdb/example_test.go @@ -40,13 +40,13 @@ func Example() { series := labels.FromStrings("foo", "bar") // Ref is 0 for the first append since we don't know the reference for the series. - ref, err := app.Append(0, series, time.Now().Unix(), 123) + ref, err := app.Append(0, series, time.Now().Unix(), 123, nil) noErr(err) // Another append for a second later. // Re-using the ref from above since it's the same series, makes append faster. time.Sleep(time.Second) - _, err = app.Append(ref, series, time.Now().Unix(), 124) + _, err = app.Append(ref, series, time.Now().Unix(), 124, nil) noErr(err) // Commit to storage. diff --git a/tsdb/head_append.go b/tsdb/head_append.go index adfd5d4bf7..536ce6a3f7 100644 --- a/tsdb/head_append.go +++ b/tsdb/head_append.go @@ -40,14 +40,14 @@ type initAppender struct { var _ storage.GetRef = &initAppender{} -func (a *initAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (a *initAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { if a.app != nil { - return a.app.Append(ref, lset, t, v) + return a.app.Append(ref, lset, t, v, nil) } a.head.initTime(t) a.app = a.head.appender() - return a.app.Append(ref, lset, t, v) + return a.app.Append(ref, lset, t, v, nil) } func (a *initAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { @@ -67,24 +67,24 @@ func (a *initAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e return a.app.AppendExemplar(ref, l, e) } -func (a *initAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a *initAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if a.app != nil { - return a.app.AppendHistogram(ref, l, t, h, fh) + return a.app.AppendHistogram(ref, l, t, h, fh, nil) } a.head.initTime(t) a.app = a.head.appender() - return a.app.AppendHistogram(ref, l, t, h, fh) + return a.app.AppendHistogram(ref, l, t, h, fh, nil) } -func (a *initAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a *initAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if a.app != nil { - return a.app.AppendHistogramCTZeroSample(ref, l, t, ct, h, fh) + return a.app.AppendHistogramCTZeroSample(ref, l, t, ct, h, fh, nil) } a.head.initTime(t) a.app = a.head.appender() - return a.app.AppendHistogramCTZeroSample(ref, l, t, ct, h, fh) + return a.app.AppendHistogramCTZeroSample(ref, l, t, ct, h, fh, nil) } func (a *initAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) { @@ -328,7 +328,7 @@ type headAppender struct { closed bool } -func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64, hints *storage.AppendHints) (storage.SeriesRef, error) { // Fail fast if OOO is disabled and the sample is out of bounds. // Otherwise a full check will be done later to decide if the sample is in-order or out-of-order. if a.oooTimeWindow == 0 && t < a.minValidTime { @@ -352,9 +352,9 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 // in commit. This code should move into Commit(). switch { case s.lastHistogramValue != nil: - return a.AppendHistogram(ref, lset, t, &histogram.Histogram{Sum: v}, nil) + return a.AppendHistogram(ref, lset, t, &histogram.Histogram{Sum: v}, nil, nil) case s.lastFloatHistogramValue != nil: - return a.AppendHistogram(ref, lset, t, nil, &histogram.FloatHistogram{Sum: v}) + return a.AppendHistogram(ref, lset, t, nil, &histogram.FloatHistogram{Sum: v}, nil) } } @@ -630,7 +630,7 @@ func (a *headAppender) AppendExemplar(ref storage.SeriesRef, lset labels.Labels, return storage.SeriesRef(s.ref), nil } -func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if !a.head.opts.EnableNativeHistograms.Load() { return 0, storage.ErrNativeHistogramsDisabled } @@ -751,7 +751,7 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels return storage.SeriesRef(s.ref), nil } -func (a *headAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, lset labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { +func (a *headAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64, h *histogram.Histogram, fh *histogram.FloatHistogram, hints *storage.AppendHints) (storage.SeriesRef, error) { if !a.head.opts.EnableNativeHistograms.Load() { return 0, storage.ErrNativeHistogramsDisabled } @@ -764,7 +764,7 @@ func (a *headAppender) AppendHistogramCTZeroSample(ref storage.SeriesRef, lset l s := a.head.series.getByID(chunks.HeadSeriesRef(ref)) if s == nil { var err error - s, created, err = a.getOrCreate(lset) + s, created, err = a.getOrCreate(l) if err != nil { return 0, err } diff --git a/tsdb/head_read_test.go b/tsdb/head_read_test.go index 6dd4c0ff55..beff329e7c 100644 --- a/tsdb/head_read_test.go +++ b/tsdb/head_read_test.go @@ -421,7 +421,7 @@ func TestHeadIndexReader_PostingsForLabelMatching(t *testing.T) { }) app := h.Appender(context.Background()) for _, s := range series { - app.Append(0, s, 0, 0) + app.Append(0, s, 0, 0, nil) } require.NoError(t, app.Commit()) diff --git a/tsdb/head_test.go b/tsdb/head_test.go index cc9daa97fe..66007f4af8 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -122,7 +122,7 @@ func BenchmarkHeadAppender_Append_Commit_ExistingSeries(b *testing.B) { for _, s := range series[:seriesCount] { var ref storage.SeriesRef for sampleIndex := int64(0); sampleIndex < samplesPerAppend; sampleIndex++ { - ref, err = app.Append(ref, s.Labels(), ts+sampleIndex, float64(ts+sampleIndex)) + ref, err = app.Append(ref, s.Labels(), ts+sampleIndex, float64(ts+sampleIndex), nil) if err != nil { return err } @@ -536,7 +536,7 @@ func TestHead_HighConcurrencyReadAndWrite(t *testing.T) { app := head.Appender(ctx) for i := 0; i < len(workerLabelSets); i++ { // We also use the timestamp as the sample value. - _, err := app.Append(0, workerLabelSets[i], int64(ts), float64(ts)) + _, err := app.Append(0, workerLabelSets[i], int64(ts), float64(ts), nil) if err != nil { return false, fmt.Errorf("Error when appending to head: %w", err) } @@ -746,14 +746,14 @@ func TestHead_WALMultiRef(t *testing.T) { require.NoError(t, head.Init(0)) app := head.Appender(context.Background()) - ref1, err := app.Append(0, labels.FromStrings("foo", "bar"), 100, 1) + ref1, err := app.Append(0, labels.FromStrings("foo", "bar"), 100, 1, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Equal(t, 1.0, prom_testutil.ToFloat64(head.metrics.chunksCreated)) // Add another sample outside chunk range to mmap a chunk. app = head.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("foo", "bar"), 1500, 2) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), 1500, 2, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Equal(t, 2.0, prom_testutil.ToFloat64(head.metrics.chunksCreated)) @@ -761,14 +761,14 @@ func TestHead_WALMultiRef(t *testing.T) { require.NoError(t, head.Truncate(1600)) app = head.Appender(context.Background()) - ref2, err := app.Append(0, labels.FromStrings("foo", "bar"), 1700, 3) + ref2, err := app.Append(0, labels.FromStrings("foo", "bar"), 1700, 3, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Equal(t, 3.0, prom_testutil.ToFloat64(head.metrics.chunksCreated)) // Add another sample outside chunk range to mmap a chunk. app = head.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("foo", "bar"), 2000, 4) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), 2000, 4, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Equal(t, 4.0, prom_testutil.ToFloat64(head.metrics.chunksCreated)) @@ -819,7 +819,7 @@ func TestHead_ActiveAppenders(t *testing.T) { // Now rollback with one sample. app = head.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("foo", "bar"), 100, 1) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), 100, 1, nil) require.NoError(t, err) require.Equal(t, 1.0, prom_testutil.ToFloat64(head.metrics.activeAppenders)) require.NoError(t, app.Rollback()) @@ -827,7 +827,7 @@ func TestHead_ActiveAppenders(t *testing.T) { // Now commit with one sample. app = head.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("foo", "bar"), 100, 1) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), 100, 1, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Equal(t, 0.0, prom_testutil.ToFloat64(head.metrics.activeAppenders)) @@ -1313,7 +1313,7 @@ func TestHeadDeleteSimple(t *testing.T) { app := head.Appender(context.Background()) for _, smpl := range smplsAll { - _, err := app.Append(0, lblsDefault, smpl.t, smpl.f) + _, err := app.Append(0, lblsDefault, smpl.t, smpl.f, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -1326,7 +1326,7 @@ func TestHeadDeleteSimple(t *testing.T) { // Add more samples. app = head.Appender(context.Background()) for _, smpl := range c.addSamples { - _, err := app.Append(0, lblsDefault, smpl.t, smpl.f) + _, err := app.Append(0, lblsDefault, smpl.t, smpl.f, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -1397,7 +1397,7 @@ func TestDeleteUntilCurMax(t *testing.T) { smpls := make([]float64, numSamples) for i := int64(0); i < numSamples; i++ { smpls[i] = rand.Float64() - _, err := app.Append(0, labels.FromStrings("a", "b"), i, smpls[i]) + _, err := app.Append(0, labels.FromStrings("a", "b"), i, smpls[i], nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -1418,7 +1418,7 @@ func TestDeleteUntilCurMax(t *testing.T) { // Add again and test for presence. app = hb.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("a", "b"), 11, 1) + _, err = app.Append(0, labels.FromStrings("a", "b"), 11, 1, nil) require.NoError(t, err) require.NoError(t, app.Commit()) q, err = NewBlockQuerier(hb, 0, 100000) @@ -1444,7 +1444,7 @@ func TestDeletedSamplesAndSeriesStillInWALAfterCheckpoint(t *testing.T) { for i := 0; i < numSamples; i++ { app := hb.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("a", "b"), int64(i), 0) + _, err := app.Append(0, labels.FromStrings("a", "b"), int64(i), 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) } @@ -1544,7 +1544,7 @@ func TestDelete_e2e(t *testing.T) { ts := rand.Int63n(300) for i := 0; i < numDatapoints; i++ { v := rand.Float64() - _, err := app.Append(0, ls, ts, v) + _, err := app.Append(0, ls, ts, v, nil) require.NoError(t, err) series = append(series, sample{ts, v, nil, nil}) ts += rand.Int63n(timeInterval) + 1 @@ -2027,7 +2027,7 @@ func TestUncommittedSamplesNotLostOnTruncate(t *testing.T) { app := h.appender() lset := labels.FromStrings("a", "1") - _, err := app.Append(0, lset, 2100, 1) + _, err := app.Append(0, lset, 2100, 1, nil) require.NoError(t, err) require.NoError(t, h.Truncate(2000)) @@ -2057,7 +2057,7 @@ func TestRemoveSeriesAfterRollbackAndTruncate(t *testing.T) { app := h.appender() lset := labels.FromStrings("a", "1") - _, err := app.Append(0, lset, 2100, 1) + _, err := app.Append(0, lset, 2100, 1, nil) require.NoError(t, err) require.NoError(t, h.Truncate(2000)) @@ -2087,7 +2087,7 @@ func TestHead_LogRollback(t *testing.T) { }() app := h.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("a", "b"), 1, 2) + _, err := app.Append(0, labels.FromStrings("a", "b"), 1, 2, nil) require.NoError(t, err) require.NoError(t, app.Rollback()) @@ -2117,7 +2117,7 @@ func TestHead_ReturnsSortedLabelValues(t *testing.T) { "__name__", fmt.Sprintf("metric_%d", i), "label", fmt.Sprintf("value_%d", j), ) - _, err := app.Append(0, lset, 2100, 1) + _, err := app.Append(0, lset, 2100, 1, nil) require.NoError(t, err) } } @@ -2382,7 +2382,7 @@ func TestNewWalSegmentOnTruncate(t *testing.T) { }() add := func(ts int64) { app := h.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("a", "b"), ts, 0) + _, err := app.Append(0, labels.FromStrings("a", "b"), ts, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) } @@ -2413,7 +2413,7 @@ func TestAddDuplicateLabelName(t *testing.T) { add := func(labels labels.Labels, labelName string) { app := h.Appender(context.Background()) - _, err := app.Append(0, labels, 0, 0) + _, err := app.Append(0, labels, 0, 0, nil) require.EqualError(t, err, fmt.Sprintf(`label name "%s" is not unique: invalid sample`, labelName)) } @@ -2477,7 +2477,7 @@ func TestMemSeriesIsolation(t *testing.T) { app = a } - _, err := app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i)) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i), nil) require.NoError(t, err) require.NoError(t, app.Commit()) h.mmapHeadChunks() @@ -2506,7 +2506,7 @@ func TestMemSeriesIsolation(t *testing.T) { // Cleanup appendIDs below 500. app := hb.appender() app.cleanupAppendIDsBelow = 500 - _, err := app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i)) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i), nil) require.NoError(t, err) require.NoError(t, app.Commit()) i++ @@ -2525,7 +2525,7 @@ func TestMemSeriesIsolation(t *testing.T) { // the only thing with appendIDs. app = hb.appender() app.cleanupAppendIDsBelow = 1000 - _, err = app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i)) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i), nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Equal(t, 999, lastValue(hb, 998)) @@ -2539,7 +2539,7 @@ func TestMemSeriesIsolation(t *testing.T) { // Cleanup appendIDs below 1001, but with a rollback. app = hb.appender() app.cleanupAppendIDsBelow = 1001 - _, err = app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i)) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i), nil) require.NoError(t, err) require.NoError(t, app.Rollback()) require.Equal(t, 1000, lastValue(hb, 999)) @@ -2575,7 +2575,7 @@ func TestMemSeriesIsolation(t *testing.T) { // Cleanup appendIDs below 1000, which means the sample buffer is // the only thing with appendIDs. app = hb.appender() - _, err = app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i)) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i), nil) i++ require.NoError(t, err) require.NoError(t, app.Commit()) @@ -2588,7 +2588,7 @@ func TestMemSeriesIsolation(t *testing.T) { // Cleanup appendIDs below 1002, but with a rollback. app = hb.appender() - _, err = app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i)) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), int64(i), float64(i), nil) require.NoError(t, err) require.NoError(t, app.Rollback()) require.Equal(t, 1001, lastValue(hb, 999)) @@ -2610,21 +2610,21 @@ func TestIsolationRollback(t *testing.T) { }() app := hb.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("foo", "bar"), 0, 0) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), 0, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Equal(t, uint64(1), hb.iso.lowWatermark()) app = hb.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("foo", "bar"), 1, 1) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), 1, 1, nil) require.NoError(t, err) - _, err = app.Append(0, labels.FromStrings("foo", "bar", "foo", "baz"), 2, 2) + _, err = app.Append(0, labels.FromStrings("foo", "bar", "foo", "baz"), 2, 2, nil) require.Error(t, err) require.NoError(t, app.Rollback()) require.Equal(t, uint64(2), hb.iso.lowWatermark()) app = hb.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("foo", "bar"), 3, 3) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), 3, 3, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Equal(t, uint64(3), hb.iso.lowWatermark(), "Low watermark should proceed to 3 even if append #2 was rolled back.") @@ -2641,18 +2641,18 @@ func TestIsolationLowWatermarkMonotonous(t *testing.T) { }() app1 := hb.Appender(context.Background()) - _, err := app1.Append(0, labels.FromStrings("foo", "bar"), 0, 0) + _, err := app1.Append(0, labels.FromStrings("foo", "bar"), 0, 0, nil) require.NoError(t, err) require.NoError(t, app1.Commit()) require.Equal(t, uint64(1), hb.iso.lowWatermark(), "Low watermark should by 1 after 1st append.") app1 = hb.Appender(context.Background()) - _, err = app1.Append(0, labels.FromStrings("foo", "bar"), 1, 1) + _, err = app1.Append(0, labels.FromStrings("foo", "bar"), 1, 1, nil) require.NoError(t, err) require.Equal(t, uint64(2), hb.iso.lowWatermark(), "Low watermark should be two, even if append is not committed yet.") app2 := hb.Appender(context.Background()) - _, err = app2.Append(0, labels.FromStrings("foo", "baz"), 1, 1) + _, err = app2.Append(0, labels.FromStrings("foo", "baz"), 1, 1, nil) require.NoError(t, err) require.NoError(t, app2.Commit()) require.Equal(t, uint64(2), hb.iso.lowWatermark(), "Low watermark should stay two because app1 is not committed yet.") @@ -2712,7 +2712,7 @@ func TestIsolationWithoutAdd(t *testing.T) { require.NoError(t, app.Commit()) app = hb.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("foo", "baz"), 1, 1) + _, err := app.Append(0, labels.FromStrings("foo", "baz"), 1, 1, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -2835,10 +2835,10 @@ func testHeadSeriesChunkRace(t *testing.T) { require.NoError(t, h.Init(0)) app := h.Appender(context.Background()) - s2, err := app.Append(0, labels.FromStrings("foo2", "bar"), 5, 0) + s2, err := app.Append(0, labels.FromStrings("foo2", "bar"), 5, 0, nil) require.NoError(t, err) for ts := int64(6); ts < 11; ts++ { - _, err = app.Append(s2, labels.EmptyLabels(), ts, 0) + _, err = app.Append(s2, labels.EmptyLabels(), ts, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -2886,7 +2886,7 @@ func TestHeadLabelNamesValuesWithMinMaxRange(t *testing.T) { app := head.Appender(ctx) for i, name := range expectedLabelNames { - _, err := app.Append(0, labels.FromStrings(name, expectedLabelValues[i]), seriesTimestamps[i], 0) + _, err := app.Append(0, labels.FromStrings(name, expectedLabelValues[i]), seriesTimestamps[i], 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -2934,7 +2934,7 @@ func TestHeadLabelValuesWithMatchers(t *testing.T) { _, err := app.Append(0, labels.FromStrings( "tens", fmt.Sprintf("value%d", i/10), "unique", fmt.Sprintf("value%d", i), - ), 100, 0) + ), 100, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -3009,14 +3009,14 @@ func TestHeadLabelNamesWithMatchers(t *testing.T) { for i := 0; i < 100; i++ { _, err := app.Append(0, labels.FromStrings( "unique", fmt.Sprintf("value%d", i), - ), 100, 0) + ), 100, 0, nil) require.NoError(t, err) if i%10 == 0 { _, err := app.Append(0, labels.FromStrings( "tens", fmt.Sprintf("value%d", i/10), "unique", fmt.Sprintf("value%d", i), - ), 100, 0) + ), 100, 0, nil) require.NoError(t, err) } @@ -3025,7 +3025,7 @@ func TestHeadLabelNamesWithMatchers(t *testing.T) { "tens", fmt.Sprintf("value%d", i/10), "twenties", fmt.Sprintf("value%d", i/20), "unique", fmt.Sprintf("value%d", i), - ), 100, 0) + ), 100, 0, nil) require.NoError(t, err) } } @@ -3080,7 +3080,7 @@ func TestHeadShardedPostings(t *testing.T) { // Append some series. app := head.Appender(ctx) for i := 0; i < 100; i++ { - _, err := app.Append(0, labels.FromStrings("unique", fmt.Sprintf("value%d", i), "const", "1"), 100, 0) + _, err := app.Append(0, labels.FromStrings("unique", fmt.Sprintf("value%d", i), "const", "1"), 100, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -3139,28 +3139,28 @@ func TestErrReuseAppender(t *testing.T) { }() app := head.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("test", "test"), 0, 0) + _, err := app.Append(0, labels.FromStrings("test", "test"), 0, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Error(t, app.Commit()) require.Error(t, app.Rollback()) app = head.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("test", "test"), 1, 0) + _, err = app.Append(0, labels.FromStrings("test", "test"), 1, 0, nil) require.NoError(t, err) require.NoError(t, app.Rollback()) require.Error(t, app.Rollback()) require.Error(t, app.Commit()) app = head.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("test", "test"), 2, 0) + _, err = app.Append(0, labels.FromStrings("test", "test"), 2, 0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) require.Error(t, app.Rollback()) require.Error(t, app.Commit()) app = head.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("test", "test"), 3, 0) + _, err = app.Append(0, labels.FromStrings("test", "test"), 3, 0, nil) require.NoError(t, err) require.NoError(t, app.Rollback()) require.Error(t, app.Commit()) @@ -3172,11 +3172,11 @@ func TestHeadMintAfterTruncation(t *testing.T) { head, _ := newTestHead(t, chunkRange, wlog.CompressionNone, false) app := head.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("a", "b"), 100, 100) + _, err := app.Append(0, labels.FromStrings("a", "b"), 100, 100, nil) require.NoError(t, err) - _, err = app.Append(0, labels.FromStrings("a", "b"), 4000, 200) + _, err = app.Append(0, labels.FromStrings("a", "b"), 4000, 200, nil) require.NoError(t, err) - _, err = app.Append(0, labels.FromStrings("a", "b"), 8000, 300) + _, err = app.Append(0, labels.FromStrings("a", "b"), 8000, 300, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -3210,7 +3210,7 @@ func TestHeadExemplars(t *testing.T) { // It is perfectly valid to add Exemplars before the current start time - // histogram buckets that haven't been update in a while could still be // exported exemplars from an hour ago. - ref, err := app.Append(0, labels.FromStrings("a", "b"), 100, 100) + ref, err := app.Append(0, labels.FromStrings("a", "b"), 100, 100, nil) require.NoError(t, err) _, err = app.AppendExemplar(ref, l, exemplar.Exemplar{ Labels: l, @@ -3238,7 +3238,7 @@ func BenchmarkHeadLabelValuesWithMatchers(b *testing.B) { "a_unique", fmt.Sprintf("value%d", i), "b_tens", fmt.Sprintf("value%d", i/(metricCount/10)), "c_ninety", fmt.Sprintf("value%d", i/(metricCount/10)/9), // "0" for the first 90%, then "1" - ), 100, 0) + ), 100, 0, nil) require.NoError(b, err) } require.NoError(b, app.Commit()) @@ -3339,7 +3339,7 @@ func TestChunkNotFoundHeadGCRace(t *testing.T) { // 7 chunks with 15s scrape interval. for i := int64(0); i <= 120*7; i++ { ts := i * DefaultBlockDuration / (4 * 120) - ref, err = app.Append(ref, labels.FromStrings("a", "b"), ts, float64(i)) + ref, err = app.Append(ref, labels.FromStrings("a", "b"), ts, float64(i), nil) require.NoError(t, err) maxt = ts } @@ -3406,7 +3406,7 @@ func TestDataMissingOnQueryDuringCompaction(t *testing.T) { // 7 chunks with 15s scrape interval. for i := int64(0); i <= 120*7; i++ { ts := i * DefaultBlockDuration / (4 * 120) - ref, err = app.Append(ref, labels.FromStrings("a", "b"), ts, float64(i)) + ref, err = app.Append(ref, labels.FromStrings("a", "b"), ts, float64(i), nil) require.NoError(t, err) maxt = ts expSamples = append(expSamples, sample{ts, float64(i), nil, nil}) @@ -3448,7 +3448,7 @@ func TestIsQuerierCollidingWithTruncation(t *testing.T) { ) for i := int64(0); i <= 3000; i++ { - ref, err = app.Append(ref, labels.FromStrings("a", "b"), i, float64(i)) + ref, err = app.Append(ref, labels.FromStrings("a", "b"), i, float64(i), nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -3496,7 +3496,7 @@ func TestWaitForPendingReadersInTimeRange(t *testing.T) { for i := int64(0); i <= 3000; i++ { ts := sampleTs(i) - ref, err = app.Append(ref, labels.FromStrings("a", "b"), ts, float64(i)) + ref, err = app.Append(ref, labels.FromStrings("a", "b"), ts, float64(i), nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -3611,12 +3611,12 @@ func testQueryOOOHeadDuringTruncate(t *testing.T, makeQuerier func(db *DB, minT, ) // Add in-order samples at every 100ms starting at 0ms. for i := int64(0); i < maxT; i += 100 { - _, err := app.Append(ref, labels.FromStrings("a", "b"), i, 0) + _, err := app.Append(ref, labels.FromStrings("a", "b"), i, 0, nil) require.NoError(t, err) } // Add out-of-order samples at every 100ms starting at 50ms. for i := int64(50); i < maxT; i += 100 { - _, err := app.Append(ref, labels.FromStrings("a", "b"), i, 0) + _, err := app.Append(ref, labels.FromStrings("a", "b"), i, 0, nil) require.NoError(t, err) } require.NoError(t, app.Commit()) @@ -3683,7 +3683,7 @@ func TestAppendHistogram(t *testing.T) { // Counter integer histograms. for _, h := range tsdbutil.GenerateTestHistograms(numHistograms) { - _, err := app.AppendHistogram(0, l, ingestTs, h, nil) + _, err := app.AppendHistogram(0, l, ingestTs, h, nil, nil) require.NoError(t, err) expHistograms = append(expHistograms, sample{t: ingestTs, h: h}) ingestTs++ @@ -3695,7 +3695,7 @@ func TestAppendHistogram(t *testing.T) { // Gauge integer histograms. for _, h := range tsdbutil.GenerateTestGaugeHistograms(numHistograms) { - _, err := app.AppendHistogram(0, l, ingestTs, h, nil) + _, err := app.AppendHistogram(0, l, ingestTs, h, nil, nil) require.NoError(t, err) expHistograms = append(expHistograms, sample{t: ingestTs, h: h}) ingestTs++ @@ -3709,7 +3709,7 @@ func TestAppendHistogram(t *testing.T) { // Counter float histograms. for _, fh := range tsdbutil.GenerateTestFloatHistograms(numHistograms) { - _, err := app.AppendHistogram(0, l, ingestTs, nil, fh) + _, err := app.AppendHistogram(0, l, ingestTs, nil, fh, nil) require.NoError(t, err) expFloatHistograms = append(expFloatHistograms, sample{t: ingestTs, fh: fh}) ingestTs++ @@ -3721,7 +3721,7 @@ func TestAppendHistogram(t *testing.T) { // Gauge float histograms. for _, fh := range tsdbutil.GenerateTestGaugeFloatHistograms(numHistograms) { - _, err := app.AppendHistogram(0, l, ingestTs, nil, fh) + _, err := app.AppendHistogram(0, l, ingestTs, nil, fh, nil) require.NoError(t, err) expFloatHistograms = append(expFloatHistograms, sample{t: ingestTs, fh: fh}) ingestTs++ @@ -3798,7 +3798,7 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { for _, h := range hists { h.NegativeSpans = h.PositiveSpans h.NegativeBuckets = h.PositiveBuckets - _, err := app.AppendHistogram(0, s1, ts, h, nil) + _, err := app.AppendHistogram(0, s1, ts, h, nil, nil) require.NoError(t, err) exp[k1] = append(exp[k1], sample{t: ts, h: h.Copy()}) ts++ @@ -3820,7 +3820,7 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { for _, h := range hists { h.NegativeSpans = h.PositiveSpans h.NegativeBuckets = h.PositiveBuckets - _, err := app.AppendHistogram(0, s1, ts, nil, h) + _, err := app.AppendHistogram(0, s1, ts, nil, h, nil) require.NoError(t, err) exp[k1] = append(exp[k1], sample{t: ts, fh: h.Copy()}) ts++ @@ -3861,7 +3861,7 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { ts++ h.NegativeSpans = h.PositiveSpans h.NegativeBuckets = h.PositiveBuckets - _, err := app.AppendHistogram(0, s2, ts, h, nil) + _, err := app.AppendHistogram(0, s2, ts, h, nil, nil) require.NoError(t, err) eh := h.Copy() if !gauge && ts > 30 && (ts-10)%20 == 1 { @@ -3875,7 +3875,7 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { // Add some float. for i := 0; i < 10; i++ { ts++ - _, err := app.Append(0, s2, ts, float64(ts)) + _, err := app.Append(0, s2, ts, float64(ts), nil) require.NoError(t, err) exp[k2] = append(exp[k2], sample{t: ts, f: float64(ts)}) } @@ -3897,7 +3897,7 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { ts++ h.NegativeSpans = h.PositiveSpans h.NegativeBuckets = h.PositiveBuckets - _, err := app.AppendHistogram(0, s2, ts, nil, h) + _, err := app.AppendHistogram(0, s2, ts, nil, h, nil) require.NoError(t, err) eh := h.Copy() if !gauge && ts > 30 && (ts-10)%20 == 1 { @@ -3911,7 +3911,7 @@ func TestHistogramInWALAndMmapChunk(t *testing.T) { // Add some float. for i := 0; i < 10; i++ { ts++ - _, err := app.Append(0, s2, ts, float64(ts)) + _, err := app.Append(0, s2, ts, float64(ts), nil) require.NoError(t, err) exp[k2] = append(exp[k2], sample{t: ts, f: float64(ts)}) } @@ -4075,17 +4075,17 @@ func TestChunkSnapshot(t *testing.T) { for ts := int64(1); ts <= 240; ts++ { val := rand.Float64() expSeries[lblStr] = append(expSeries[lblStr], sample{ts, val, nil, nil}) - ref, err := app.Append(0, lbls, ts, val) + ref, err := app.Append(0, lbls, ts, val, nil) require.NoError(t, err) hist := histograms[int(ts)] expHist[lblsHistStr] = append(expHist[lblsHistStr], sample{ts, 0, hist, nil}) - _, err = app.AppendHistogram(0, lblsHist, ts, hist, nil) + _, err = app.AppendHistogram(0, lblsHist, ts, hist, nil, nil) require.NoError(t, err) floatHist := floatHistogram[int(ts)] expFloatHist[lblsFloatHistStr] = append(expFloatHist[lblsFloatHistStr], sample{ts, 0, nil, floatHist}) - _, err = app.AppendHistogram(0, lblsFloatHist, ts, nil, floatHist) + _, err = app.AppendHistogram(0, lblsFloatHist, ts, nil, floatHist, nil) require.NoError(t, err) // Add an exemplar and to create multiple WAL records. @@ -4149,17 +4149,17 @@ func TestChunkSnapshot(t *testing.T) { for ts := int64(241); ts <= 480; ts++ { val := rand.Float64() expSeries[lblStr] = append(expSeries[lblStr], sample{ts, val, nil, nil}) - ref, err := app.Append(0, lbls, ts, val) + ref, err := app.Append(0, lbls, ts, val, nil) require.NoError(t, err) hist := histograms[int(ts)] expHist[lblsHistStr] = append(expHist[lblsHistStr], sample{ts, 0, hist, nil}) - _, err = app.AppendHistogram(0, lblsHist, ts, hist, nil) + _, err = app.AppendHistogram(0, lblsHist, ts, hist, nil, nil) require.NoError(t, err) floatHist := floatHistogram[int(ts)] expFloatHist[lblsFloatHistStr] = append(expFloatHist[lblsFloatHistStr], sample{ts, 0, nil, floatHist}) - _, err = app.AppendHistogram(0, lblsFloatHist, ts, nil, floatHist) + _, err = app.AppendHistogram(0, lblsFloatHist, ts, nil, floatHist, nil) require.NoError(t, err) // Add an exemplar and to create multiple WAL records. @@ -4250,7 +4250,7 @@ func TestSnapshotError(t *testing.T) { // Add a sample. app := head.Appender(context.Background()) lbls := labels.FromStrings("foo", "bar") - _, err := app.Append(0, lbls, 99, 99) + _, err := app.Append(0, lbls, 99, 99, nil) require.NoError(t, err) // Add histograms @@ -4259,10 +4259,10 @@ func TestSnapshotError(t *testing.T) { lblsHist := labels.FromStrings("hist", "bar") lblsFloatHist := labels.FromStrings("floathist", "bar") - _, err = app.AppendHistogram(0, lblsHist, 99, hist, nil) + _, err = app.AppendHistogram(0, lblsHist, 99, hist, nil, nil) require.NoError(t, err) - _, err = app.AppendHistogram(0, lblsFloatHist, 99, nil, floatHist) + _, err = app.AppendHistogram(0, lblsFloatHist, 99, nil, floatHist, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -4365,14 +4365,14 @@ func TestHistogramMetrics(t *testing.T) { l := labels.FromStrings("a", fmt.Sprintf("b%d", x)) for i, h := range tsdbutil.GenerateTestHistograms(numHistograms) { app := head.Appender(context.Background()) - _, err := app.AppendHistogram(0, l, int64(i), h, nil) + _, err := app.AppendHistogram(0, l, int64(i), h, nil, nil) require.NoError(t, err) require.NoError(t, app.Commit()) expHSamples++ } for i, fh := range tsdbutil.GenerateTestFloatHistograms(numHistograms) { app := head.Appender(context.Background()) - _, err := app.AppendHistogram(0, l, int64(numHistograms+i), nil, fh) + _, err := app.AppendHistogram(0, l, int64(numHistograms+i), nil, fh, nil) require.NoError(t, err) require.NoError(t, app.Commit()) expHSamples++ @@ -4490,16 +4490,16 @@ func testHistogramStaleSampleHelper(t *testing.T, floatHistogram bool) { for _, h := range tsdbutil.GenerateTestHistograms(numHistograms) { var err error if floatHistogram { - _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat(nil)) + _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat(nil), nil) expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), fh: h.ToFloat(nil)}) } else { - _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h, nil) + _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h, nil, nil) expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), h: h}) } require.NoError(t, err) } // +1 so that delta-of-delta is not 0. - _, err := app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN)) + _, err := app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN), nil) require.NoError(t, err) if floatHistogram { expHistograms = append(expHistograms, timedHistogram{t: 100*int64(len(expHistograms)) + 1, fh: &histogram.FloatHistogram{Sum: math.Float64frombits(value.StaleNaN)}}) @@ -4521,10 +4521,10 @@ func testHistogramStaleSampleHelper(t *testing.T, floatHistogram bool) { for _, h := range tsdbutil.GenerateTestHistograms(2 * numHistograms)[numHistograms:] { var err error if floatHistogram { - _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat(nil)) + _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), nil, h.ToFloat(nil), nil) expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), fh: h.ToFloat(nil)}) } else { - _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h, nil) + _, err = app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h, nil, nil) expHistograms = append(expHistograms, timedHistogram{t: 100 * int64(len(expHistograms)), h: h}) } require.NoError(t, err) @@ -4533,7 +4533,7 @@ func testHistogramStaleSampleHelper(t *testing.T, floatHistogram bool) { app = head.Appender(context.Background()) // +1 so that delta-of-delta is not 0. - _, err = app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN)) + _, err = app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN), nil) require.NoError(t, err) if floatHistogram { expHistograms = append(expHistograms, timedHistogram{t: 100*int64(len(expHistograms)) + 1, fh: &histogram.FloatHistogram{Sum: math.Float64frombits(value.StaleNaN)}}) @@ -4568,9 +4568,9 @@ func TestHistogramCounterResetHeader(t *testing.T) { app := head.Appender(context.Background()) var err error if floatHisto { - _, err = app.AppendHistogram(0, l, ts, nil, h.ToFloat(nil)) + _, err = app.AppendHistogram(0, l, ts, nil, h.ToFloat(nil), nil) } else { - _, err = app.AppendHistogram(0, l, ts, h.Copy(), nil) + _, err = app.AppendHistogram(0, l, ts, h.Copy(), nil, nil) } require.NoError(t, err) require.NoError(t, app.Commit()) @@ -4690,9 +4690,9 @@ func TestOOOHistogramCounterResetHeaders(t *testing.T) { app := head.Appender(context.Background()) var err error if floatHisto { - _, err = app.AppendHistogram(0, l, ts, nil, h.ToFloat(nil)) + _, err = app.AppendHistogram(0, l, ts, nil, h.ToFloat(nil), nil) } else { - _, err = app.AppendHistogram(0, l, ts, h.Copy(), nil) + _, err = app.AppendHistogram(0, l, ts, h.Copy(), nil, nil) } require.NoError(t, err) require.NoError(t, app.Commit()) @@ -4949,9 +4949,9 @@ func TestAppendingDifferentEncodingToSameSeries(t *testing.T) { for _, s := range a.samples { var err error if s.H() != nil || s.FH() != nil { - _, err = app.AppendHistogram(0, lbls, s.T(), s.H(), s.FH()) + _, err = app.AppendHistogram(0, lbls, s.T(), s.H(), s.FH(), nil) } else { - _, err = app.Append(0, lbls, s.T(), s.F()) + _, err = app.Append(0, lbls, s.T(), s.F(), nil) } require.Equal(t, a.err, err) } @@ -5074,7 +5074,7 @@ func TestChunkSnapshotTakenAfterIncompleteSnapshot(t *testing.T) { // Add some samples for the snapshot. app := head.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("foo", "bar"), 10, 10) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), 10, 10, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -5290,7 +5290,7 @@ func TestHeadInit_DiscardChunksWithUnsupportedEncoding(t *testing.T) { var seriesRef storage.SeriesRef var err error for i := 0; i < 400; i++ { - seriesRef, err = app.Append(0, seriesLabels, int64(i), float64(i)) + seriesRef, err = app.Append(0, seriesLabels, int64(i), float64(i), nil) require.NoError(t, err) } @@ -5303,7 +5303,7 @@ func TestHeadInit_DiscardChunksWithUnsupportedEncoding(t *testing.T) { app = h.Appender(ctx) for i := 700; i < 1200; i++ { - _, err := app.Append(0, seriesLabels, int64(i), float64(i)) + _, err := app.Append(0, seriesLabels, int64(i), float64(i), nil) require.NoError(t, err) } @@ -5376,7 +5376,7 @@ func TestMmapPanicAfterMmapReplayCorruption(t *testing.T) { interval := DefaultBlockDuration / (4 * 120) app := h.Appender(context.Background()) for i := 0; i < 250; i++ { - ref, err = app.Append(ref, lbls, lastTs, float64(lastTs)) + ref, err = app.Append(ref, lbls, lastTs, float64(lastTs), nil) lastTs += interval if i%10 == 0 { require.NoError(t, app.Commit()) @@ -5439,7 +5439,7 @@ func TestReplayAfterMmapReplayError(t *testing.T) { app := h.Appender(context.Background()) var ref storage.SeriesRef for i := 0; i < numSamples; i++ { - ref, err = app.Append(ref, lbls, lastTs, float64(lastTs)) + ref, err = app.Append(ref, lbls, lastTs, float64(lastTs), nil) expSamples = append(expSamples, sample{t: lastTs, f: float64(lastTs)}) require.NoError(t, err) lastTs += itvl @@ -5648,7 +5648,7 @@ func TestGaugeHistogramWALAndChunkHeader(t *testing.T) { appendHistogram := func(h *histogram.Histogram) { ts++ app := head.Appender(context.Background()) - _, err := app.AppendHistogram(0, l, ts, h.Copy(), nil) + _, err := app.AppendHistogram(0, l, ts, h.Copy(), nil, nil) require.NoError(t, err) require.NoError(t, app.Commit()) } @@ -5724,7 +5724,7 @@ func TestGaugeFloatHistogramWALAndChunkHeader(t *testing.T) { appendHistogram := func(h *histogram.FloatHistogram) { ts++ app := head.Appender(context.Background()) - _, err := app.AppendHistogram(0, l, ts, nil, h.Copy()) + _, err := app.AppendHistogram(0, l, ts, nil, h.Copy(), nil) require.NoError(t, err) require.NoError(t, app.Commit()) } @@ -5793,7 +5793,7 @@ func TestSnapshotAheadOfWALError(t *testing.T) { head.opts.EnableMemorySnapshotOnShutdown = true // Add a sample to fill WAL. app := head.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings("foo", "bar"), 10, 10) + _, err := app.Append(0, labels.FromStrings("foo", "bar"), 10, 10, nil) require.NoError(t, err) require.NoError(t, app.Commit()) @@ -5817,7 +5817,7 @@ func TestSnapshotAheadOfWALError(t *testing.T) { require.NoError(t, err) // Add a sample to fill WAL. app = head.Appender(context.Background()) - _, err = app.Append(0, labels.FromStrings("foo", "bar"), 10, 10) + _, err = app.Append(0, labels.FromStrings("foo", "bar"), 10, 10, nil) require.NoError(t, err) require.NoError(t, app.Commit()) lastSegment, _, _ := w.LastSegmentAndOffset() @@ -5862,7 +5862,7 @@ func BenchmarkCuttingHeadHistogramChunks(b *testing.B) { b.ResetTimer() for _, s := range samples { - _, err := a.AppendHistogram(0, lbls, ts, s, nil) + _, err := a.AppendHistogram(0, lbls, ts, s, nil, nil) require.NoError(b, err) } } @@ -5980,10 +5980,10 @@ func TestCuttingNewHeadChunks(t *testing.T) { for i := 0; i < tc.numTotalSamples; i++ { if tc.floatValFunc != nil { - _, err := a.Append(0, lbls, ts, tc.floatValFunc(i)) + _, err := a.Append(0, lbls, ts, tc.floatValFunc(i), nil) require.NoError(t, err) } else if tc.histValFunc != nil { - _, err := a.AppendHistogram(0, lbls, ts, tc.histValFunc(i), nil) + _, err := a.AppendHistogram(0, lbls, ts, tc.histValFunc(i), nil, nil) require.NoError(t, err) } @@ -6045,7 +6045,7 @@ func TestHeadDetectsDuplicateSampleAtSizeLimit(t *testing.T) { vals := []float64{math.MaxFloat64, 0x00} // Use the worst case scenario for the XOR encoding. Otherwise we hit the sample limit before the size limit. for i := 0; i < numSamples; i++ { ts := baseTS + int64(i/2)*10000 - a.Append(0, labels.FromStrings("foo", "bar"), ts, vals[(i/2)%len(vals)]) + a.Append(0, labels.FromStrings("foo", "bar"), ts, vals[(i/2)%len(vals)], nil) err = a.Commit() require.NoError(t, err) a = h.Appender(context.Background()) @@ -6082,19 +6082,19 @@ func TestWALSampleAndExemplarOrder(t *testing.T) { }{ "float sample": { appendF: func(app storage.Appender, ts int64) (storage.SeriesRef, error) { - return app.Append(0, lbls, ts, 1.0) + return app.Append(0, lbls, ts, 1.0, nil) }, expectedType: reflect.TypeOf([]record.RefSample{}), }, "histogram sample": { appendF: func(app storage.Appender, ts int64) (storage.SeriesRef, error) { - return app.AppendHistogram(0, lbls, ts, tsdbutil.GenerateTestHistogram(1), nil) + return app.AppendHistogram(0, lbls, ts, tsdbutil.GenerateTestHistogram(1), nil, nil) }, expectedType: reflect.TypeOf([]record.RefHistogramSample{}), }, "float histogram sample": { appendF: func(app storage.Appender, ts int64) (storage.SeriesRef, error) { - return app.AppendHistogram(0, lbls, ts, nil, tsdbutil.GenerateTestFloatHistogram(1)) + return app.AppendHistogram(0, lbls, ts, nil, tsdbutil.GenerateTestFloatHistogram(1), nil) }, expectedType: reflect.TypeOf([]record.RefFloatHistogramSample{}), }, @@ -6142,7 +6142,7 @@ func TestHeadCompactionWhileAppendAndCommitExemplar(t *testing.T) { h, _ := newTestHead(t, DefaultBlockDuration, wlog.CompressionNone, false) app := h.Appender(context.Background()) lbls := labels.FromStrings("foo", "bar") - ref, err := app.Append(0, lbls, 1, 1) + ref, err := app.Append(0, lbls, 1, 1, nil) require.NoError(t, err) app.Commit() // Not adding a sample here to trigger the fault. @@ -6260,7 +6260,7 @@ func TestHeadAppender_AppendFloatWithSameTimestampAsPreviousHistogram(t *testing { // Append a float 10.0 @ 1_000 app := head.Appender(context.Background()) - _, err := app.Append(0, ls, 1_000, 10.0) + _, err := app.Append(0, ls, 1_000, 10.0, nil) require.NoError(t, err) require.NoError(t, app.Commit()) } @@ -6269,13 +6269,13 @@ func TestHeadAppender_AppendFloatWithSameTimestampAsPreviousHistogram(t *testing // Append a float histogram @ 2_000 app := head.Appender(context.Background()) h := tsdbutil.GenerateTestHistogram(1) - _, err := app.AppendHistogram(0, ls, 2_000, h, nil) + _, err := app.AppendHistogram(0, ls, 2_000, h, nil, nil) require.NoError(t, err) require.NoError(t, app.Commit()) } app := head.Appender(context.Background()) - _, err := app.Append(0, ls, 2_000, 10.0) + _, err := app.Append(0, ls, 2_000, 10.0, nil) require.Error(t, err) require.ErrorIs(t, err, storage.NewDuplicateHistogramToFloatErr(2_000, 10.0)) } @@ -6479,15 +6479,15 @@ func TestHeadAppender_AppendCT(t *testing.T) { if sample.fSample != 0 { _, err := a.AppendCTZeroSample(0, lbls, sample.ts, sample.ct) require.NoError(t, err) - _, err = a.Append(0, lbls, sample.ts, sample.fSample) + _, err = a.Append(0, lbls, sample.ts, sample.fSample, nil) require.NoError(t, err) } // Append histograms if it's a histogram test case if sample.h != nil || sample.fh != nil { - ref, err := a.AppendHistogramCTZeroSample(0, lbls, sample.ts, sample.ct, sample.h, sample.fh) + ref, err := a.AppendHistogramCTZeroSample(0, lbls, sample.ts, sample.ct, sample.h, sample.fh, nil) require.NoError(t, err) - _, err = a.AppendHistogram(ref, lbls, sample.ts, sample.h, sample.fh) + _, err = a.AppendHistogram(ref, lbls, sample.ts, sample.h, sample.fh, nil) require.NoError(t, err) } } @@ -6532,11 +6532,11 @@ func TestHeadAppendHistogramAndCommitConcurrency(t *testing.T) { testCases := map[string]func(storage.Appender, int) error{ "integer histogram": func(app storage.Appender, i int) error { - _, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar", "serial", strconv.Itoa(i)), 1, h, nil) + _, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar", "serial", strconv.Itoa(i)), 1, h, nil, nil) return err }, "float histogram": func(app storage.Appender, i int) error { - _, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar", "serial", strconv.Itoa(i)), 1, nil, fh) + _, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar", "serial", strconv.Itoa(i)), 1, nil, fh, nil) return err }, } diff --git a/tsdb/querier_bench_test.go b/tsdb/querier_bench_test.go index 33dca1284d..81800b7d47 100644 --- a/tsdb/querier_bench_test.go +++ b/tsdb/querier_bench_test.go @@ -43,7 +43,7 @@ func BenchmarkQuerier(b *testing.B) { app := h.Appender(context.Background()) addSeries := func(l labels.Labels) { - app.Append(0, l, 0, 0) + app.Append(0, l, 0, 0, nil) } for n := 0; n < 10; n++ { @@ -298,7 +298,7 @@ func benchmarkSelect(b *testing.B, queryable storage.Queryable, numSeries int, s func BenchmarkQuerierSelect(b *testing.B) { numSeries := 1000000 h, db := createHeadForBenchmarkSelect(b, numSeries, func(app storage.Appender, i int) { - _, err := app.Append(0, labels.FromStrings("foo", "bar", "i", fmt.Sprintf("%d%s", i, postingsBenchSuffix)), int64(i), 0) + _, err := app.Append(0, labels.FromStrings("foo", "bar", "i", fmt.Sprintf("%d%s", i, postingsBenchSuffix)), int64(i), 0, nil) if err != nil { b.Fatal(err) } @@ -336,11 +336,11 @@ func BenchmarkQuerierSelectWithOutOfOrder(b *testing.B) { numSeries := 1000000 _, db := createHeadForBenchmarkSelect(b, numSeries, func(app storage.Appender, i int) { l := labels.FromStrings("foo", "bar", "i", fmt.Sprintf("%d%s", i, postingsBenchSuffix)) - ref, err := app.Append(0, l, int64(i+1), 0) + ref, err := app.Append(0, l, int64(i+1), 0, nil) if err != nil { b.Fatal(err) } - _, err = app.Append(ref, l, int64(i), 1) // Out of order sample + _, err = app.Append(ref, l, int64(i), 1, nil) // Out of order sample if err != nil { b.Fatal(err) } diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 77772937a7..fb18c6a88b 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -506,7 +506,7 @@ func TestBlockQuerier_AgainstHeadWithOpenChunks(t *testing.T) { for _, s := range testData { for _, chk := range s.chunks { for _, sample := range chk { - _, err = app.Append(0, labels.FromMap(s.lset), sample.t, sample.f) + _, err = app.Append(0, labels.FromMap(s.lset), sample.t, sample.f, nil) require.NoError(t, err) } } @@ -2686,12 +2686,12 @@ func TestPostingsForMatchers(t *testing.T) { }() app := h.Appender(context.Background()) - app.Append(0, labels.FromStrings("n", "1"), 0, 0) - app.Append(0, labels.FromStrings("n", "1", "i", "a"), 0, 0) - app.Append(0, labels.FromStrings("n", "1", "i", "b"), 0, 0) - app.Append(0, labels.FromStrings("n", "1", "i", "\n"), 0, 0) - app.Append(0, labels.FromStrings("n", "2"), 0, 0) - app.Append(0, labels.FromStrings("n", "2.5"), 0, 0) + app.Append(0, labels.FromStrings("n", "1"), 0, 0, nil) + app.Append(0, labels.FromStrings("n", "1", "i", "a"), 0, 0, nil) + app.Append(0, labels.FromStrings("n", "1", "i", "b"), 0, 0, nil) + app.Append(0, labels.FromStrings("n", "1", "i", "\n"), 0, 0, nil) + app.Append(0, labels.FromStrings("n", "2"), 0, 0, nil) + app.Append(0, labels.FromStrings("n", "2.5"), 0, 0, nil) require.NoError(t, app.Commit()) cases := []struct { @@ -3103,7 +3103,7 @@ func appendSeries(t *testing.T, ctx context.Context, wg *sync.WaitGroup, h *Head for i := 0; ctx.Err() == nil; i++ { app := h.Appender(context.Background()) - _, err := app.Append(0, labels.FromStrings(labels.MetricName, "metric", "seq", strconv.Itoa(i), "always_0", "0"), 0, 0) + _, err := app.Append(0, labels.FromStrings(labels.MetricName, "metric", "seq", strconv.Itoa(i), "always_0", "0"), 0, 0, nil) require.NoError(t, err) err = app.Commit() require.NoError(t, err) @@ -3485,7 +3485,7 @@ func BenchmarkHeadChunkQuerier(b *testing.B) { require.NoError(b, app.Commit()) app = db.Appender(context.Background()) } - _, err := app.Append(0, lbls, int64(i*15)*time.Second.Milliseconds(), float64(i*100)) + _, err := app.Append(0, lbls, int64(i*15)*time.Second.Milliseconds(), float64(i*100), nil) require.NoError(b, err) } } @@ -3530,7 +3530,7 @@ func BenchmarkHeadQuerier(b *testing.B) { require.NoError(b, app.Commit()) app = db.Appender(context.Background()) } - _, err := app.Append(0, lbls, int64(i*15)*time.Second.Milliseconds(), float64(i*100)) + _, err := app.Append(0, lbls, int64(i*15)*time.Second.Milliseconds(), float64(i*100), nil) require.NoError(b, err) } } @@ -3596,7 +3596,7 @@ func TestQueryWithDeletedHistograms(t *testing.T) { for i := 0; i < 100; i++ { h, fh := tc(i) - seriesRef, err = appender.AppendHistogram(seriesRef, lbs, int64(i), h, fh) + seriesRef, err = appender.AppendHistogram(seriesRef, lbs, int64(i), h, fh, nil) require.NoError(t, err) } @@ -3656,12 +3656,12 @@ func TestQueryWithOneChunkCompletelyDeleted(t *testing.T) { // Create an int histogram chunk with samples between 0 - 20 and 30 - 40. for i := 0; i < 20; i++ { h := tsdbutil.GenerateTestHistogram(1) - seriesRef, err = appender.AppendHistogram(seriesRef, lbs, int64(i), h, nil) + seriesRef, err = appender.AppendHistogram(seriesRef, lbs, int64(i), h, nil, nil) require.NoError(t, err) } for i := 30; i < 40; i++ { h := tsdbutil.GenerateTestHistogram(1) - seriesRef, err = appender.AppendHistogram(seriesRef, lbs, int64(i), h, nil) + seriesRef, err = appender.AppendHistogram(seriesRef, lbs, int64(i), h, nil, nil) require.NoError(t, err) } @@ -3669,7 +3669,7 @@ func TestQueryWithOneChunkCompletelyDeleted(t *testing.T) { // type from int histograms so a new chunk is created. for i := 60; i < 100; i++ { fh := tsdbutil.GenerateTestFloatHistogram(1) - seriesRef, err = appender.AppendHistogram(seriesRef, lbs, int64(i), nil, fh) + seriesRef, err = appender.AppendHistogram(seriesRef, lbs, int64(i), nil, fh, nil) require.NoError(t, err) } diff --git a/tsdb/testutil.go b/tsdb/testutil.go index 03587f4e2c..1b34a4da2f 100644 --- a/tsdb/testutil.go +++ b/tsdb/testutil.go @@ -53,7 +53,7 @@ var sampleTypeScenarios = map[string]sampleTypeScenario{ sampleType: sampleMetricTypeFloat, appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) { s := sample{t: ts, f: float64(value)} - ref, err := appender.Append(0, lbls, ts, s.f) + ref, err := appender.Append(0, lbls, ts, s.f, nil) return ref, s, err }, sampleFunc: func(ts, value int64) sample { @@ -64,7 +64,7 @@ var sampleTypeScenarios = map[string]sampleTypeScenario{ sampleType: sampleMetricTypeHistogram, appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) { s := sample{t: ts, h: tsdbutil.GenerateTestHistogram(int(value))} - ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil) + ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil, nil) return ref, s, err }, sampleFunc: func(ts, value int64) sample { @@ -75,7 +75,7 @@ var sampleTypeScenarios = map[string]sampleTypeScenario{ sampleType: sampleMetricTypeHistogram, appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) { s := sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(value))} - ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh) + ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh, nil) return ref, s, err }, sampleFunc: func(ts, value int64) sample { @@ -86,7 +86,7 @@ var sampleTypeScenarios = map[string]sampleTypeScenario{ sampleType: sampleMetricTypeHistogram, appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) { s := sample{t: ts, h: tsdbutil.GenerateTestGaugeHistogram(int(value))} - ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil) + ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil, nil) return ref, s, err }, sampleFunc: func(ts, value int64) sample { @@ -97,7 +97,7 @@ var sampleTypeScenarios = map[string]sampleTypeScenario{ sampleType: sampleMetricTypeHistogram, appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, sample, error) { s := sample{t: ts, fh: tsdbutil.GenerateTestGaugeFloatHistogram(int(value))} - ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh) + ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh, nil) return ref, s, err }, sampleFunc: func(ts, value int64) sample { diff --git a/tsdb/tsdbblockutil.go b/tsdb/tsdbblockutil.go index b49757223f..a4d8bcec6c 100644 --- a/tsdb/tsdbblockutil.go +++ b/tsdb/tsdbblockutil.go @@ -70,13 +70,13 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger * switch typ { case chunkenc.ValFloat: t, v := it.At() - ref, err = app.Append(ref, lset, t, v) + ref, err = app.Append(ref, lset, t, v, nil) case chunkenc.ValHistogram: t, h := it.AtHistogram(nil) - ref, err = app.AppendHistogram(ref, lset, t, h, nil) + ref, err = app.AppendHistogram(ref, lset, t, h, nil, nil) case chunkenc.ValFloatHistogram: t, fh := it.AtFloatHistogram(nil) - ref, err = app.AppendHistogram(ref, lset, t, nil, fh) + ref, err = app.AppendHistogram(ref, lset, t, nil, fh, nil) default: return "", fmt.Errorf("unknown sample type %s", typ.String()) } diff --git a/web/federate_test.go b/web/federate_test.go index 056a95d67f..37d0688410 100644 --- a/web/federate_test.go +++ b/web/federate_test.go @@ -347,14 +347,14 @@ func TestFederationWithNativeHistograms(t *testing.T) { var err error switch i { case 0, 3: - _, err = app.Append(0, l, 100*60*1000, float64(i*100)) + _, err = app.Append(0, l, 100*60*1000, float64(i*100), nil) expVec = append(expVec, promql.Sample{ T: 100 * 60 * 1000, F: float64(i * 100), Metric: expL, }) case 4: - _, err = app.AppendHistogram(0, l, 100*60*1000, histWithoutZeroBucket.Copy(), nil) + _, err = app.AppendHistogram(0, l, 100*60*1000, histWithoutZeroBucket.Copy(), nil, nil) expVec = append(expVec, promql.Sample{ T: 100 * 60 * 1000, H: histWithoutZeroBucket.ToFloat(nil), @@ -363,7 +363,7 @@ func TestFederationWithNativeHistograms(t *testing.T) { default: hist.ZeroCount++ hist.Count++ - _, err = app.AppendHistogram(0, l, 100*60*1000, hist.Copy(), nil) + _, err = app.AppendHistogram(0, l, 100*60*1000, hist.Copy(), nil, nil) expVec = append(expVec, promql.Sample{ T: 100 * 60 * 1000, H: hist.ToFloat(nil),