From c0707c8ba1a5437745bacd9332ef2ae3be181d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Mon, 4 May 2026 14:51:53 +0200 Subject: [PATCH] search: add BuildStartTimestamp to IndexMeta (#124035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records when the bleve index was originally created (start of the from-scratch build), as distinct from the snapshot upload time. Sourced from the bleve index's internal buildInfo, which is preserved across reopens and snapshot round-trips, so periodic re-uploads of a long-lived index re-emit the original build-start time. Snapshots produced before this change have a zero-value field; readers must treat zero as "unknown" and fall back to other criteria. Plumbing only — no reader uses the field yet. --- .../unified/search/bleve_snapshot_upload.go | 12 ++++- .../search/bleve_snapshot_upload_test.go | 40 +++++++++++++++++ .../unified/search/remote_index_store.go | 8 ++++ .../unified/search/remote_index_store_test.go | 45 +++++++++++++++++++ 4 files changed, 103 insertions(+), 2 deletions(-) diff --git a/pkg/storage/unified/search/bleve_snapshot_upload.go b/pkg/storage/unified/search/bleve_snapshot_upload.go index 4de55e34924..9053ecdacf2 100644 --- a/pkg/storage/unified/search/bleve_snapshot_upload.go +++ b/pkg/storage/unified/search/bleve_snapshot_upload.go @@ -111,10 +111,18 @@ func (b *bleveBackend) uploadSnapshot(ctx context.Context, key resource.Namespac return fmt.Errorf("reading snapshot build info: %w", biErr) } - uploadKey, err = b.opts.Snapshot.Store.UploadIndex(ctx, key, stagingDir, IndexMeta{ + meta := IndexMeta{ GrafanaBuildVersion: bi.BuildVersion, LatestResourceVersion: rv, - }) + } + // bi.BuildTime is the original index creation time; it survives reopens and + // downloads, so periodic re-uploads keep the original build-start time. + // Guard zero so legacy indexes without BuildTime stay zero in the manifest. + if bi.BuildTime > 0 { + meta.BuildStartTimestamp = time.Unix(bi.BuildTime, 0).UTC() + } + + uploadKey, err = b.opts.Snapshot.Store.UploadIndex(ctx, key, stagingDir, meta) if err != nil { return fmt.Errorf("uploading snapshot: %w", err) } diff --git a/pkg/storage/unified/search/bleve_snapshot_upload_test.go b/pkg/storage/unified/search/bleve_snapshot_upload_test.go index f9023005817..742930ce4bc 100644 --- a/pkg/storage/unified/search/bleve_snapshot_upload_test.go +++ b/pkg/storage/unified/search/bleve_snapshot_upload_test.go @@ -160,12 +160,19 @@ func TestUploadSnapshot_Success(t *testing.T) { store := &uploadTestStore{} be, _ := newTestBleveBackend(t, SnapshotOptions{Store: store}) key := newTestNsResource() + beforeBuild := time.Now().Add(-time.Second).Truncate(time.Second) idx := newUploadTestIndex(t, be, key, 42) require.NoError(t, be.uploadSnapshot(context.Background(), key, idx)) assert.Equal(t, int32(1), store.uploadCalls.Load()) assert.Equal(t, int64(42), store.uploadMeta.LatestResourceVersion) assert.Equal(t, be.opts.BuildVersion, store.uploadMeta.GrafanaBuildVersion) + // BuildStartTimestamp must be populated from the index's internal build + // info (set by newBleveIndex), not left zero. Compare with second-level + // granularity since buildInfo persists Unix seconds. + assert.False(t, store.uploadMeta.BuildStartTimestamp.IsZero(), "BuildStartTimestamp should be set") + assert.False(t, store.uploadMeta.BuildStartTimestamp.Before(beforeBuild), + "BuildStartTimestamp %s should be at or after %s", store.uploadMeta.BuildStartTimestamp, beforeBuild) assert.NotEmpty(t, store.uploaded) snapshotParent := filepath.Join(be.opts.Root, "snapshots", resourceSubPath(key)) @@ -174,6 +181,39 @@ func TestUploadSnapshot_Success(t *testing.T) { assert.Empty(t, entries) } +// TestUploadSnapshot_PreservesOriginalBuildStartTime verifies that periodic +// re-uploads of a long-lived index re-emit the original build-start time +// (carried in the bleve index's internal buildInfo), not the upload time. +func TestUploadSnapshot_PreservesOriginalBuildStartTime(t *testing.T) { + store := &uploadTestStore{} + be, _ := newTestBleveBackend(t, SnapshotOptions{Store: store}) + key := newTestNsResource() + + resourceDir := be.getResourceDir(key) + require.NoError(t, os.MkdirAll(resourceDir, 0o750)) + + originalBuildTime := time.Now().Add(-72 * time.Hour).Truncate(time.Second) + index, err := newBleveIndex( + filepath.Join(resourceDir, formatIndexName(time.Now())), + bleve.NewIndexMapping(), + originalBuildTime, + be.opts.BuildVersion, + nil, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = index.Close() }) + require.NoError(t, index.Index("dash-1", map[string]string{"title": "Production Overview"})) + require.NoError(t, setRV(index, 42)) + + wrapped := be.newBleveIndex(key, index, indexStorageFile, nil, nil, nil, nil, be.log) + wrapped.resourceVersion.Store(42) + + require.NoError(t, be.uploadSnapshot(context.Background(), key, wrapped)) + require.Equal(t, int32(1), store.uploadCalls.Load()) + assert.Equal(t, originalBuildTime.UTC(), store.uploadMeta.BuildStartTimestamp, + "periodic re-upload should preserve the original build-start time") +} + func TestUploadSnapshot_LockContention(t *testing.T) { store := &uploadTestStore{lockErr: errLockHeld} be, _ := newTestBleveBackend(t, SnapshotOptions{Store: store}) diff --git a/pkg/storage/unified/search/remote_index_store.go b/pkg/storage/unified/search/remote_index_store.go index 1e293fdf296..4d3fa7a1479 100644 --- a/pkg/storage/unified/search/remote_index_store.go +++ b/pkg/storage/unified/search/remote_index_store.go @@ -43,6 +43,14 @@ type IndexMeta struct { GrafanaBuildVersion string `json:"grafana_build_version"` // UploadTimestamp is when the snapshot was uploaded. UploadTimestamp time.Time `json:"upload_timestamp"` + // BuildStartTimestamp is when the bleve index was originally created + // (start of the from-scratch build that produced it). Persisted across + // periodic re-uploads of the same index, so it always describes the + // underlying data, not the most recent upload. + // + // Zero-value means "unknown" — snapshots produced before this field was + // introduced. Readers fall back to other criteria in that case. + BuildStartTimestamp time.Time `json:"build_start_timestamp,omitempty"` // LatestResourceVersion is the latest resource version included in the index. LatestResourceVersion int64 `json:"latest_resource_version"` // Files maps relative file paths to their sizes in bytes. diff --git a/pkg/storage/unified/search/remote_index_store_test.go b/pkg/storage/unified/search/remote_index_store_test.go index e6a7c2426a6..cc5ff93b381 100644 --- a/pkg/storage/unified/search/remote_index_store_test.go +++ b/pkg/storage/unified/search/remote_index_store_test.go @@ -96,9 +96,11 @@ func TestRemoteIndexStore_UploadDownloadBleveIndex(t *testing.T) { ns := newTestNsResource() srcDir := createTestBleveIndex(t) + buildStart := time.Now().Add(-2 * time.Hour).UTC().Truncate(time.Second) meta := IndexMeta{ GrafanaBuildVersion: "11.0.0", LatestResourceVersion: 99, + BuildStartTimestamp: buildStart, } indexKey, err := store.UploadIndex(ctx, ns, srcDir, meta) @@ -110,6 +112,16 @@ func TestRemoteIndexStore_UploadDownloadBleveIndex(t *testing.T) { require.NoError(t, err) assert.Equal(t, meta.GrafanaBuildVersion, gotMeta.GrafanaBuildVersion) assert.Equal(t, meta.LatestResourceVersion, gotMeta.LatestResourceVersion) + assert.True(t, gotMeta.BuildStartTimestamp.Equal(buildStart), + "BuildStartTimestamp should round-trip: got %s, want %s", gotMeta.BuildStartTimestamp, buildStart) + + // ListIndexes must surface the same value. + listed, err := store.ListIndexes(ctx, ns) + require.NoError(t, err) + require.Contains(t, listed, indexKey) + assert.True(t, listed[indexKey].BuildStartTimestamp.Equal(buildStart), + "BuildStartTimestamp should round-trip via ListIndexes: got %s, want %s", + listed[indexKey].BuildStartTimestamp, buildStart) // Open and query the downloaded index idx, err := bleve.Open(destDir) @@ -131,6 +143,39 @@ func TestRemoteIndexStore_UploadDownloadBleveIndex(t *testing.T) { require.NoError(t, idx.Close()) } +// TestRemoteIndexStore_ListIndexes_LegacyMetaWithoutBuildStartTime verifies +// that a snapshot manifest produced before the BuildStartTimestamp field was +// introduced is still accepted by ListIndexes and surfaces a zero-value +// BuildStartTimestamp. Readers must treat zero as "unknown". +func TestRemoteIndexStore_ListIndexes_LegacyMetaWithoutBuildStartTime(t *testing.T) { + ctx := context.Background() + bucket := memblob.OpenBucket(nil) + t.Cleanup(func() { _ = bucket.Close() }) + store := newTestRemoteIndexStore(t, bucket) + ns := newTestNsResource() + indexKey := ulid.Make() + + // Hand-crafted manifest with no build_start_timestamp field at all, + // mirroring the on-disk shape of legacy snapshots. + legacyManifest := []byte(`{ + "grafana_build_version": "11.0.0", + "upload_timestamp": "2024-01-01T00:00:00Z", + "latest_resource_version": 42, + "files": {"store/root.bolt": 1} + }`) + pfx := indexPrefix(ns, indexKey.String()) + require.NoError(t, bucket.WriteAll(ctx, pfx+snapshotManifestFile, legacyManifest, nil)) + + listed, err := store.ListIndexes(ctx, ns) + require.NoError(t, err) + require.Contains(t, listed, indexKey) + assert.True(t, listed[indexKey].BuildStartTimestamp.IsZero(), + "legacy manifest should decode to zero-valued BuildStartTimestamp, got %s", + listed[indexKey].BuildStartTimestamp) + assert.Equal(t, "11.0.0", listed[indexKey].GrafanaBuildVersion) + assert.Equal(t, int64(42), listed[indexKey].LatestResourceVersion) +} + func TestRemoteIndexStore_ListAndDeleteIndexes(t *testing.T) { store := newTestRemoteIndexStore(t, testBucket(t)) ctx := context.Background()