From 405813f250019ff7fd8c15c83bf418cfd83af625 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Nov 2025 17:09:07 +0100 Subject: [PATCH] repository: fix LookupBlobSize to also report pending blobs --- internal/repository/index/master_index.go | 38 ++++++------------- .../repository/index/master_index_test.go | 13 +------ internal/repository/repository.go | 4 +- 3 files changed, 15 insertions(+), 40 deletions(-) diff --git a/internal/repository/index/master_index.go b/internal/repository/index/master_index.go index 62ccc4f71..f410ebf61 100644 --- a/internal/repository/index/master_index.go +++ b/internal/repository/index/master_index.go @@ -16,13 +16,13 @@ import ( // MasterIndex is a collection of indexes and IDs of chunks that are in the process of being saved. type MasterIndex struct { idx []*Index - pendingBlobs restic.BlobSet + pendingBlobs map[restic.BlobHandle]uint idxMutex sync.RWMutex } // NewMasterIndex creates a new master index. func NewMasterIndex() *MasterIndex { - mi := &MasterIndex{pendingBlobs: restic.NewBlobSet()} + mi := &MasterIndex{pendingBlobs: make(map[restic.BlobHandle]uint)} mi.clear() return mi } @@ -46,10 +46,16 @@ func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) { } // LookupSize queries all known Indexes for the ID and returns the first match. +// Also returns true if the ID is pending. func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() + // also return true if blob is pending + if size, ok := mi.pendingBlobs[bh]; ok { + return size, true + } + for _, idx := range mi.idx { if size, found := idx.LookupSize(bh); found { return size, found @@ -63,13 +69,13 @@ func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) { // Before doing so it checks if this blob is already known. // Returns true if adding was successful and false if the blob // was already known -func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool { +func (mi *MasterIndex) AddPending(bh restic.BlobHandle, size uint) bool { mi.idxMutex.Lock() defer mi.idxMutex.Unlock() // Check if blob is pending or in index - if mi.pendingBlobs.Has(bh) { + if _, ok := mi.pendingBlobs[bh]; ok { return false } @@ -80,30 +86,10 @@ func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool { } // really not known -> insert - mi.pendingBlobs.Insert(bh) + mi.pendingBlobs[bh] = size return true } -// Has queries all known Indexes for the ID and returns the first match. -// Also returns true if the ID is pending. -func (mi *MasterIndex) Has(bh restic.BlobHandle) bool { - mi.idxMutex.RLock() - defer mi.idxMutex.RUnlock() - - // also return true if blob is pending - if mi.pendingBlobs.Has(bh) { - return true - } - - for _, idx := range mi.idx { - if idx.Has(bh) { - return true - } - } - - return false -} - // IDs returns the IDs of all indexes contained in the index. func (mi *MasterIndex) IDs() restic.IDSet { mi.idxMutex.RLock() @@ -165,7 +151,7 @@ func (mi *MasterIndex) storePack(id restic.ID, blobs []restic.Blob) { // delete blobs from pending for _, blob := range blobs { - mi.pendingBlobs.Delete(restic.BlobHandle{Type: blob.Type, ID: blob.ID}) + delete(mi.pendingBlobs, restic.BlobHandle{Type: blob.Type, ID: blob.ID}) } for _, idx := range mi.idx { diff --git a/internal/repository/index/master_index_test.go b/internal/repository/index/master_index_test.go index edf2067b9..98cfe9ac6 100644 --- a/internal/repository/index/master_index_test.go +++ b/internal/repository/index/master_index_test.go @@ -74,9 +74,6 @@ func TestMasterIndex(t *testing.T) { mIdx.Insert(idx2) // test idInIdx1 - found := mIdx.Has(bhInIdx1) - rtest.Equals(t, true, found) - blobs := mIdx.Lookup(bhInIdx1) rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) @@ -85,9 +82,6 @@ func TestMasterIndex(t *testing.T) { rtest.Equals(t, uint(10), size) // test idInIdx2 - found = mIdx.Has(bhInIdx2) - rtest.Equals(t, true, found) - blobs = mIdx.Lookup(bhInIdx2) rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) @@ -96,9 +90,6 @@ func TestMasterIndex(t *testing.T) { rtest.Equals(t, uint(200), size) // test idInIdx12 - found = mIdx.Has(bhInIdx12) - rtest.Equals(t, true, found) - blobs = mIdx.Lookup(bhInIdx12) rtest.Equals(t, 2, len(blobs)) @@ -121,8 +112,6 @@ func TestMasterIndex(t *testing.T) { rtest.Equals(t, uint(80), size) // test not in index - found = mIdx.Has(restic.BlobHandle{ID: restic.NewRandomID(), Type: restic.TreeBlob}) - rtest.Assert(t, !found, "Expected no blobs when fetching with a random id") blobs = mIdx.Lookup(restic.NewRandomBlobHandle()) rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id") _, found = mIdx.LookupSize(restic.NewRandomBlobHandle()) @@ -521,7 +510,7 @@ func TestRewriteOversizedIndex(t *testing.T) { // verify that blobs are still in the index for _, blob := range blobs { - found := mi2.Has(blob.BlobHandle) + _, found := mi2.LookupSize(blob.BlobHandle) rtest.Assert(t, found, "blob %v missing after rewrite", blob.ID) } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index bb9c6c3ba..d0da2e108 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -640,7 +640,7 @@ func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.Pack return r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) } -// LookupBlobSize returns the size of blob id. +// LookupBlobSize returns the size of blob id. Also returns pending blobs. func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bool) { return r.idx.LookupSize(restic.BlobHandle{Type: tpe, ID: id}) } @@ -968,7 +968,7 @@ func (r *Repository) saveBlob(ctx context.Context, t restic.BlobType, buf []byte } // first try to add to pending blobs; if not successful, this blob is already known - known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t}) + known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t}, uint(len(buf))) // only save when needed or explicitly told if !known || storeDuplicate {