upstream/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-02-03 04:20:45 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

repository: fix LookupBlobSize to also report pending blobs

Browse source
This commit is contained in:
Michael Eischer 2025-11-23 17:09:07 +01:00
parent 05364500b6
commit 405813f250
3 changed files with 15 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
internal/repository/index/master_index.go
View file

@ -16,13 +16,13 @@ import (
// MasterIndex is a collection of indexes and IDs of chunks that are in the process of being saved.
type MasterIndex struct {
idx []*Index
pendingBlobs restic.BlobSet
pendingBlobs map[restic.BlobHandle]uint
idxMutex sync.RWMutex
}
// NewMasterIndex creates a new master index.
func NewMasterIndex() *MasterIndex {
mi := &MasterIndex{pendingBlobs: restic.NewBlobSet()}
mi := &MasterIndex{pendingBlobs: make(map[restic.BlobHandle]uint)}
mi.clear()
return mi
}
@ -46,10 +46,16 @@ func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) {
}
// LookupSize queries all known Indexes for the ID and returns the first match.
// Also returns true if the ID is pending.
func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) {
mi.idxMutex.RLock()
defer mi.idxMutex.RUnlock()
// also return true if blob is pending
if size, ok := mi.pendingBlobs[bh]; ok {
return size, true
}
for _, idx := range mi.idx {
if size, found := idx.LookupSize(bh); found {
return size, found
@ -63,13 +69,13 @@ func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) {
// Before doing so it checks if this blob is already known.
// Returns true if adding was successful and false if the blob
// was already known
func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool {
func (mi *MasterIndex) AddPending(bh restic.BlobHandle, size uint) bool {
mi.idxMutex.Lock()
defer mi.idxMutex.Unlock()
// Check if blob is pending or in index
if mi.pendingBlobs.Has(bh) {
if _, ok := mi.pendingBlobs[bh]; ok {
return false
}
@ -80,30 +86,10 @@ func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool {
}
// really not known -> insert
mi.pendingBlobs.Insert(bh)
mi.pendingBlobs[bh] = size
return true
}
// Has queries all known Indexes for the ID and returns the first match.
// Also returns true if the ID is pending.
func (mi *MasterIndex) Has(bh restic.BlobHandle) bool {
mi.idxMutex.RLock()
defer mi.idxMutex.RUnlock()
// also return true if blob is pending
if mi.pendingBlobs.Has(bh) {
return true
}
for _, idx := range mi.idx {
if idx.Has(bh) {
return true
}
}
return false
}
// IDs returns the IDs of all indexes contained in the index.
func (mi *MasterIndex) IDs() restic.IDSet {
mi.idxMutex.RLock()
@ -165,7 +151,7 @@ func (mi *MasterIndex) storePack(id restic.ID, blobs []restic.Blob) {
// delete blobs from pending
for _, blob := range blobs {
mi.pendingBlobs.Delete(restic.BlobHandle{Type: blob.Type, ID: blob.ID})
delete(mi.pendingBlobs, restic.BlobHandle{Type: blob.Type, ID: blob.ID})
}
for _, idx := range mi.idx {

13
internal/repository/index/master_index_test.go
View file

@ -74,9 +74,6 @@ func TestMasterIndex(t *testing.T) {
mIdx.Insert(idx2)
// test idInIdx1
found := mIdx.Has(bhInIdx1)
rtest.Equals(t, true, found)
blobs := mIdx.Lookup(bhInIdx1)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
@ -85,9 +82,6 @@ func TestMasterIndex(t *testing.T) {
rtest.Equals(t, uint(10), size)
// test idInIdx2
found = mIdx.Has(bhInIdx2)
rtest.Equals(t, true, found)
blobs = mIdx.Lookup(bhInIdx2)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
@ -96,9 +90,6 @@ func TestMasterIndex(t *testing.T) {
rtest.Equals(t, uint(200), size)
// test idInIdx12
found = mIdx.Has(bhInIdx12)
rtest.Equals(t, true, found)
blobs = mIdx.Lookup(bhInIdx12)
rtest.Equals(t, 2, len(blobs))
@ -121,8 +112,6 @@ func TestMasterIndex(t *testing.T) {
rtest.Equals(t, uint(80), size)
// test not in index
found = mIdx.Has(restic.BlobHandle{ID: restic.NewRandomID(), Type: restic.TreeBlob})
rtest.Assert(t, !found, "Expected no blobs when fetching with a random id")
blobs = mIdx.Lookup(restic.NewRandomBlobHandle())
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
_, found = mIdx.LookupSize(restic.NewRandomBlobHandle())
@ -521,7 +510,7 @@ func TestRewriteOversizedIndex(t *testing.T) {
// verify that blobs are still in the index
for _, blob := range blobs {
found := mi2.Has(blob.BlobHandle)
_, found := mi2.LookupSize(blob.BlobHandle)
rtest.Assert(t, found, "blob %v missing after rewrite", blob.ID)
}

4
internal/repository/repository.go
View file

@ -640,7 +640,7 @@ func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.Pack
return r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id})
}
// LookupBlobSize returns the size of blob id.
// LookupBlobSize returns the size of blob id. Also returns pending blobs.
func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bool) {
return r.idx.LookupSize(restic.BlobHandle{Type: tpe, ID: id})
}
@ -968,7 +968,7 @@ func (r *Repository) saveBlob(ctx context.Context, t restic.BlobType, buf []byte
}
// first try to add to pending blobs; if not successful, this blob is already known
known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t})
known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t}, uint(len(buf)))
// only save when needed or explicitly told
if !known || storeDuplicate {