diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 996e79e6a..3abb0b218 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -249,7 +249,7 @@ func (arch *Archiver) trackItem(item string, previous, current *data.Node, s Ite } // nodeFromFileInfo returns the restic node from an os.FileInfo. -func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) { +func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, deviceIdMap deviceIdMapper, ignoreXattrListError bool) (*data.Node, error) { node, err := meta.ToNode(ignoreXattrListError, func(format string, args ...any) { _ = arch.error(filename, fmt.Errorf(format, args...)) }) @@ -269,6 +269,15 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig node.DeviceID = 0 } } + + if feature.Flag.Enabled(feature.VirtualDeviceId) { + virtualDeviceID, ok := deviceIdMap.GetVirtualId(node.DeviceID) + if !ok { + err = errors.Errorf("A virtual device ID for device %v was not found", node.DeviceID) + return node, arch.error(filename, err) + } + node.DeviceID = virtualDeviceID + } // overwrite name to match that within the snapshot node.Name = path.Base(snPath) // do not filter error for nodes of irregular or invalid type @@ -307,10 +316,10 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { // saveDir stores a directory in the repo and returns the node. snPath is the // path within the current snapshot. -func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous data.TreeNodeIterator, complete fileCompleteFunc) (d futureNode, err error) { +func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous data.TreeNodeIterator, deviceIdMap deviceIdMapper, complete fileCompleteFunc) (d futureNode, err error) { debug.Log("%v %v", snPath, dir) - treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta) + treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta, deviceIdMap) if err != nil { return futureNode{}, err } @@ -334,7 +343,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me return futureNode{}, err } snItem := join(snPath, name) - fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode) + fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode, deviceIdMap) // return error early if possible if err != nil { @@ -359,13 +368,13 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me return fn, nil } -func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (node *data.Node, names []string, err error) { +func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File, deviceIdMap deviceIdMapper) (node *data.Node, names []string, err error) { err = meta.MakeReadable() if err != nil { return nil, nil, fmt.Errorf("openfile for readdirnames failed: %w", err) } - node, err = arch.nodeFromFileInfo(snPath, dir, meta, false) + node, err = arch.nodeFromFileInfo(snPath, dir, meta, deviceIdMap, false) if err != nil { return nil, nil, err } @@ -449,7 +458,7 @@ func (arch *Archiver) allBlobsPresent(previous *data.Node) bool { // Errors and completion needs to be handled by the caller. // // snPath is the path within the current snapshot. -func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *data.Node) (fn futureNode, excluded bool, err error) { +func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *data.Node, deviceIdMap deviceIdMapper) (fn futureNode, excluded bool, err error) { start := time.Now() debug.Log("%v target %q, previous %v", snPath, target, previous) @@ -516,7 +525,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous debug.Log("%v hasn't changed, using old list of blobs", target) arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start)) arch.CompleteBlob(previous.Size) - node, err := arch.nodeFromFileInfo(snPath, target, meta, false) + node, err := arch.nodeFromFileInfo(snPath, target, meta, deviceIdMap, false) if err != nil { return futureNode{}, false, err } @@ -584,7 +593,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous return futureNode{}, false, err } - fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree, + fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree, deviceIdMap, func(node *data.Node, stats ItemStats) { arch.trackItem(snItem, previous, node, stats, time.Since(start)) }) @@ -600,7 +609,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous default: debug.Log(" %v other", target) - node, err := arch.nodeFromFileInfo(snPath, target, meta, false) + node, err := arch.nodeFromFileInfo(snPath, target, meta, deviceIdMap, false) if err != nil { return futureNode{}, false, err } @@ -652,7 +661,7 @@ func join(elem ...string) string { // saveTree stores a Tree in the repo, returned is the tree. snPath is the path // within the current snapshot. -func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous data.TreeNodeIterator, complete fileCompleteFunc) (futureNode, int, error) { +func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous data.TreeNodeIterator, deviceIdMap deviceIdMapper, complete fileCompleteFunc) (futureNode, int, error) { var node *data.Node if snPath != "/" { @@ -661,7 +670,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, } var err error - node, err = arch.dirPathToNode(snPath, atree.FileInfoPath) + node, err = arch.dirPathToNode(snPath, atree.FileInfoPath, deviceIdMap) if err != nil { return futureNode{}, 0, err } @@ -694,7 +703,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, if err != nil { return futureNode{}, 0, err } - fn, excluded, err := arch.save(ctx, pathname, subatree.Path, oldNode) + fn, excluded, err := arch.save(ctx, pathname, subatree.Path, oldNode, deviceIdMap) if err != nil { err = arch.error(subatree.Path, err) @@ -728,7 +737,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, } // not a leaf node, archive subtree - fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *data.Node, is ItemStats) { + fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, deviceIdMap, func(n *data.Node, is ItemStats) { arch.trackItem(snItem, oldNode, n, is, time.Since(start)) }) if err != nil { @@ -747,7 +756,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, return fn, len(nodes), nil } -func (arch *Archiver) dirPathToNode(snPath, target string) (node *data.Node, err error) { +func (arch *Archiver) dirPathToNode(snPath, target string, deviceIdMap deviceIdMapper) (node *data.Node, err error) { meta, err := arch.FS.OpenFile(target, 0, true) if err != nil { return nil, err @@ -762,7 +771,7 @@ func (arch *Archiver) dirPathToNode(snPath, target string) (node *data.Node, err debug.Log("%v, reading dir node data from %v", snPath, target) // in some cases reading xattrs for directories above the backup source is not allowed // thus ignore errors for such folders. - node, err = arch.nodeFromFileInfo(snPath, target, meta, true) + node, err = arch.nodeFromFileInfo(snPath, target, meta, deviceIdMap, true) if err != nil { return nil, err } @@ -842,11 +851,12 @@ func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) dat } // runWorkers starts the worker pools, which are stopped when the context is cancelled. -func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync) { +func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, readOnlyMapper deviceIdMapper) { arch.fileSaver = newFileSaver(ctx, wg, uploader, arch.Repo.Config().ChunkerPolynomial, - arch.Options.ReadConcurrency) + arch.Options.ReadConcurrency, + readOnlyMapper) arch.fileSaver.CompleteBlob = arch.CompleteBlob arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo @@ -881,12 +891,13 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps err = arch.Repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, wgCtx := errgroup.WithContext(ctx) start := time.Now() + deviceIdMap := newMutableDeviceIdMapper() wg.Go(func() error { - arch.runWorkers(wgCtx, wg, uploader) + arch.runWorkers(wgCtx, wg, uploader, deviceIdMap.ReadOnlyMapper()) debug.Log("starting snapshot") - fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *data.Node, is ItemStats) { + fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), deviceIdMap, func(_ *data.Node, is ItemStats) { arch.trackItem("/", nil, nil, is, time.Since(start)) }) if err != nil { diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index b13734655..7175a6e33 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -58,7 +58,8 @@ func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) completeReading := func() { completeReadingCallback = true @@ -221,9 +222,10 @@ func TestArchiverSave(t *testing.T) { var fnr futureNodeResult err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) - node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil) + node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil, deviceIdMap) if err != nil { t.Fatal(err) } @@ -298,9 +300,10 @@ func TestArchiverSaveReaderFS(t *testing.T) { var fnr futureNodeResult err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) - node, excluded, err := arch.save(ctx, "/", filename, nil) + node, excluded, err := arch.save(ctx, "/", filename, nil, deviceIdMap) t.Logf("Save returned %v %v", node, err) if err != nil { t.Fatal(err) @@ -852,10 +855,11 @@ func TestArchiverSaveDir(t *testing.T) { var treeID restic.ID err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) meta, err := testFS.OpenFile(test.target, fs.O_NOFOLLOW, true) rtest.OK(t, err) - ft, err := arch.saveDir(ctx, "/", test.target, meta, nil, nil) + ft, err := arch.saveDir(ctx, "/", test.target, meta, nil, deviceIdMap, nil) rtest.OK(t, err) rtest.OK(t, meta.Close()) @@ -914,10 +918,11 @@ func TestArchiverSaveDirIncremental(t *testing.T) { var fnr futureNodeResult err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) meta, err := testFS.OpenFile(tempdir, fs.O_NOFOLLOW, true) rtest.OK(t, err) - ft, err := arch.saveDir(ctx, "/", tempdir, meta, nil, nil) + ft, err := arch.saveDir(ctx, "/", tempdir, meta, nil, deviceIdMap, nil) rtest.OK(t, err) rtest.OK(t, meta.Close()) @@ -1104,14 +1109,15 @@ func TestArchiverSaveTree(t *testing.T) { var treeID restic.ID err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) atree, err := newTree(testFS, test.targets) if err != nil { t.Fatal(err) } - fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil) + fn, _, err := arch.saveTree(ctx, "/", atree, nil, deviceIdMap, nil) if err != nil { t.Fatal(err) } @@ -2453,10 +2459,11 @@ func TestRacyFileTypeSwap(t *testing.T) { t.Logf("archiver error as expected for %v: %v", item, err) return err } - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) // fs.Track will panic if the file was not closed - _, excluded, err := arch.save(ctx, "/", tempfile, nil) + _, excluded, err := arch.save(ctx, "/", tempfile, nil, deviceIdMap) rtest.Assert(t, err != nil && strings.Contains(err.Error(), "changed type, refusing to archive"), "save() returned wrong error: %v", err) tpe := "file" if dirError { @@ -2499,7 +2506,8 @@ func TestMetadataBackupErrorFiltering(t *testing.T) { } // check that errors from reading extended metadata are properly filtered - node, err := arch.nodeFromFileInfo("file", filename+"invalid", nonExistNoder, false) + deviceIdMap := newMutableDeviceIdMapper() + node, err := arch.nodeFromFileInfo("file", filename+"invalid", nonExistNoder, deviceIdMap, false) rtest.Assert(t, node != nil, "node is missing") rtest.Assert(t, err == replacementErr, "expected %v got %v", replacementErr, err) rtest.Assert(t, filteredErr != nil, "missing inner error") @@ -2510,12 +2518,72 @@ func TestMetadataBackupErrorFiltering(t *testing.T) { node: &data.Node{Type: data.NodeTypeIrregular}, err: fmt.Errorf(`unsupported file type "irregular"`), } - node, err = arch.nodeFromFileInfo("file", filename, nonExistNoder, false) + node, err = arch.nodeFromFileInfo("file", filename, nonExistNoder, deviceIdMap, false) rtest.Assert(t, node != nil, "node is missing") rtest.Assert(t, filteredErr == nil, "error for irregular node should not have been filtered") rtest.Assert(t, strings.Contains(err.Error(), "irregular"), "unexpected error %q does not warn about irregular file mode", err) } +func TestVirtualDeviceIdOnlyAffectsHardlinks(t *testing.T) { + // This tests the scenario where both DeviceIdForHardlinks and VirtualDeviceId are set + // In this scenario, due to DeviceIdForHardlinks, all device ID's should be zero, except for hardlinks + // Then, due to VirtualDeviceId, hardlink device ids should be replaced with virtual ids + defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)() + defer feature.TestSetFlag(t, feature.Flag, feature.VirtualDeviceId, true)() + + repo := repository.TestRepository(t) + arch := New(repo, fs.Local{}, Options{}) + deviceIdMap := newMutableDeviceIdMapper() + + normalFile := &mockToNoder{ + node: &data.Node{Type: data.NodeTypeFile, Links: 1, DeviceID: 42}, + err: nil, + } + node, err := arch.nodeFromFileInfo("f", "f", normalFile, deviceIdMap, false) + rtest.OK(t, err) + if node.DeviceID != 0 { + t.Errorf("normal file (Links=1) with both flags: got DeviceID %d, want 0", node.DeviceID) + } + + hardlink := &mockToNoder{ + node: &data.Node{Type: data.NodeTypeFile, Links: 2, DeviceID: 123}, + err: nil, + } + node, err = arch.nodeFromFileInfo("g", "g", hardlink, deviceIdMap, false) + rtest.OK(t, err) + if node.DeviceID != 1 { + t.Errorf("hardlink (Links>1) with both flags: got DeviceID %d, want 1", node.DeviceID) + } +} + +func TestVirtualDeviceId(t *testing.T) { + defer feature.TestSetFlag(t, feature.Flag, feature.VirtualDeviceId, true)() + + repo := repository.TestRepository(t) + arch := New(repo, fs.Local{}, Options{}) + deviceIdMap := newMutableDeviceIdMapper() + + fileDev100 := &mockToNoder{ + node: &data.Node{Type: data.NodeTypeFile, Links: 1, DeviceID: 100}, + err: nil, + } + node1, err := arch.nodeFromFileInfo("a", "a", fileDev100, deviceIdMap, false) + rtest.OK(t, err) + if node1.DeviceID != 1 { + t.Errorf("first file (device 100): got DeviceID %d, want 1", node1.DeviceID) + } + + fileDev200 := &mockToNoder{ + node: &data.Node{Type: data.NodeTypeFile, Links: 1, DeviceID: 200}, + err: nil, + } + node2, err := arch.nodeFromFileInfo("b", "b", fileDev200, deviceIdMap, false) + rtest.OK(t, err) + if node2.DeviceID != 2 { + t.Errorf("second file (device 200): got DeviceID %d, want 2", node2.DeviceID) + } +} + func TestIrregularFile(t *testing.T) { files := TestDir{ "testfile": TestFile{ @@ -2545,7 +2613,8 @@ func TestIrregularFile(t *testing.T) { defer cancel() arch := New(repo, fs.Track{FS: override}, Options{}) - _, excluded, err := arch.save(ctx, "/", tempfile, nil) + deviceIdMap := newMutableDeviceIdMapper() + _, excluded, err := arch.save(ctx, "/", tempfile, nil, deviceIdMap) if err == nil { t.Fatalf("Save() should have failed") } @@ -2595,7 +2664,8 @@ func TestDisappearedFile(t *testing.T) { // the subsequent file.Stat() call. Thus test both cases. for _, errorOnOpen := range []bool{false, true} { arch := New(repo, fs.Track{FS: &missingFS{FS: &fs.Local{}, errorOnOpen: errorOnOpen}}, Options{}) - _, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "testdir"), nil) + deviceIdMap := newMutableDeviceIdMapper() + _, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "testdir"), nil, deviceIdMap) rtest.OK(t, err) rtest.Assert(t, excluded, "testfile should have been excluded") } diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 84e175d82..526da8fcd 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -24,14 +24,16 @@ type fileSaver struct { ch chan<- saveFileJob + deviceIdMap deviceIdMapper + CompleteBlob func(bytes uint64) - NodeFromFileInfo func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) + NodeFromFileInfo func(snPath, filename string, meta ToNoder, deviceIdMap deviceIdMapper, ignoreXattrListError bool) (*data.Node, error) } // newFileSaver returns a new file saver. A worker pool with fileWorkers is // started, it is stopped when ctx is cancelled. -func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, pol chunker.Pol, fileWorkers uint) *fileSaver { +func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, pol chunker.Pol, fileWorkers uint, readOnlyMapper deviceIdMapper) *fileSaver { ch := make(chan saveFileJob) debug.Log("new file saver with %v file workers", fileWorkers) @@ -40,6 +42,7 @@ func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobS saveFilePool: newBufferPool(chunker.MaxSize), pol: pol, ch: ch, + deviceIdMap: readOnlyMapper, CompleteBlob: func(uint64) {}, } @@ -148,7 +151,7 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat debug.Log("%v", snPath) - node, err := s.NodeFromFileInfo(snPath, target, f, false) + node, err := s.NodeFromFileInfo(snPath, target, f, s.deviceIdMap, false) if err != nil { _ = f.Close() completeError(err) diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index 4dbf78548..44303a326 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -40,8 +40,9 @@ func startFileSaver(ctx context.Context, t testing.TB, _ fs.FS) (*fileSaver, *mo } saver := &mockSaver{saved: make(map[string]int)} - s := newFileSaver(ctx, wg, saver, pol, workers) - s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) { + deviceIdMap := newMutableDeviceIdMapper().ReadOnlyMapper() + s := newFileSaver(ctx, wg, saver, pol, workers, deviceIdMap) + s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, deviceIdMap deviceIdMapper, ignoreXattrListError bool) (*data.Node, error) { return meta.ToNode(ignoreXattrListError, t.Logf) } diff --git a/internal/archiver/virtual_device_id.go b/internal/archiver/virtual_device_id.go new file mode 100644 index 000000000..b8aff9510 --- /dev/null +++ b/internal/archiver/virtual_device_id.go @@ -0,0 +1,80 @@ +package archiver + +// Virtual device ID mapping: converts real device IDs into a stable virtual +// numbering 1..n, where assignment order is determined by first-seen. This +// keeps snapshot trees stable across runs regardless of which devices appear +// or in what order. +// +// Concurrency: only the goroutine that created the mapper via +// newMutableDeviceIdMapper() may call GetVirtualId on that mapper. For that +// goroutine, GetVirtualId may allocate a new virtual ID for an unseen device. +// All other goroutines must use the result of ReadOnlyMapper(). On the +// read-only mapper, GetVirtualId returns (0, false) for any device ID not yet +// seen by the mutable owner. +// +// See deviceIdMapper and newMutableDeviceIdMapper. + +import ( + "sync" + + "github.com/restic/restic/internal/debug" +) + +// deviceIdMapper maps real device IDs to stable virtual IDs. Implementations +// are either mutable (may assign new virtual IDs) or read-only (lookup only). +type deviceIdMapper interface { + GetVirtualId(realDeviceID uint64) (uint64, bool) + ReadOnlyMapper() deviceIdMapper +} + +// newMutableDeviceIdMapper returns a mapper that may assign new virtual IDs. +// Only the creating goroutine may call GetVirtualId on the returned value. +func newMutableDeviceIdMapper() deviceIdMapper { + m := &deviceIdMap{ + count: 0, + cache: &vIdCache{}, + } + m.cache.set(0, 0) + return m +} + +// vIdCache is a concurrent read-only view of realID -> virtualID; GetVirtualId returns (0, false) when missing. +type vIdCache sync.Map + +func (v *vIdCache) GetVirtualId(realDeviceID uint64) (uint64, bool) { + if id, ok := (*sync.Map)(v).Load(realDeviceID); ok { + return id.(uint64), true + } + return 0, false +} + +func (v *vIdCache) set(realDeviceID uint64, virtualDeviceID uint64) { + (*sync.Map)(v).Store(realDeviceID, virtualDeviceID) +} + +func (v *vIdCache) ReadOnlyMapper() deviceIdMapper { + return v +} + +// deviceIdMap is the mutable mapper; only its owner goroutine may call GetVirtualId. +type deviceIdMap struct { + count uint64 + cache *vIdCache +} + +// GetVirtualId returns the virtual ID for realDeviceID, allocating one (first-seen order) if needed. +func (d *deviceIdMap) GetVirtualId(realDeviceID uint64) (uint64, bool) { + id, ok := d.cache.GetVirtualId(realDeviceID) + if !ok { + id = d.count + 1 + d.cache.set(realDeviceID, id) + d.count = id + debug.Log("Mapped deviceId %v to virtualId %v", realDeviceID, id) + } + return id, true +} + +// ReadOnlyMapper returns a concurrent-safe view for other goroutines; they must use this, not GetVirtualId on d. +func (d *deviceIdMap) ReadOnlyMapper() deviceIdMapper { + return d.cache +} diff --git a/internal/archiver/virtual_device_id_test.go b/internal/archiver/virtual_device_id_test.go new file mode 100644 index 000000000..1bd51be3f --- /dev/null +++ b/internal/archiver/virtual_device_id_test.go @@ -0,0 +1,86 @@ +package archiver + +import ( + "sync" + "testing" +) + +func TestVirtualIdFirstSeenOrder(t *testing.T) { + m := newMutableDeviceIdMapper() + + // Real 0 is pre-mapped to virtual 0 + if vid, ok := m.GetVirtualId(0); !ok || vid != 0 { + t.Fatalf("real 0: got (%d, %v), want (0, true)", vid, ok) + } + + // First-seen order: 100 -> 1, 200 -> 2, 100 again -> 1, 300 -> 3 + cases := []struct { + realID uint64 + want uint64 + }{ + {100, 1}, + {200, 2}, + {100, 1}, + {300, 3}, + } + for _, c := range cases { + vid, ok := m.GetVirtualId(c.realID) + if !ok || vid != c.want { + t.Errorf("real %d: got (%d, %v), want (%d, true)", c.realID, vid, ok, c.want) + } + } +} + +func TestReadOnlyMapperUnseenReturnsFalse(t *testing.T) { + m := newMutableDeviceIdMapper() + m.GetVirtualId(42) + ro := m.ReadOnlyMapper() + + if vid, ok := ro.GetVirtualId(999); ok || vid != 0 { + t.Errorf("unseen 999: got (%d, %v), want (0, false)", vid, ok) + } +} + +func TestReadOnlyMapperSeenReturnsId(t *testing.T) { + m := newMutableDeviceIdMapper() + vidOwner, _ := m.GetVirtualId(100) + ro := m.ReadOnlyMapper() + + vidRo, ok := ro.GetVirtualId(100) + if !ok || vidRo != vidOwner { + t.Errorf("read-only for seen 100: got (%d, %v), want (%d, true)", vidRo, ok, vidOwner) + } +} + +func TestReadOnlyMapperConcurrent(t *testing.T) { + m := newMutableDeviceIdMapper() + ro := m.ReadOnlyMapper() + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = ro.GetVirtualId(1) + _, _ = ro.GetVirtualId(999) + }() + } + + m.GetVirtualId(1) + wg.Wait() +} + +func TestReadOnlyMapperSeesUpdatesFromOwner(t *testing.T) { + m := newMutableDeviceIdMapper() + ro := m.ReadOnlyMapper() + + if _, ok := ro.GetVirtualId(1); ok { + t.Error("read-only saw 1 before owner") + } + + m.GetVirtualId(1) + vid, ok := ro.GetVirtualId(1) + if !ok || vid != 1 { + t.Errorf("after owner added 1: got (%d, %v), want (1, true)", vid, ok) + } +} diff --git a/internal/feature/registry.go b/internal/feature/registry.go index a7368fa75..0d68a9f75 100644 --- a/internal/feature/registry.go +++ b/internal/feature/registry.go @@ -9,6 +9,7 @@ const ( DeprecateLegacyIndex FlagName = "deprecate-legacy-index" DeprecateS3LegacyLayout FlagName = "deprecate-s3-legacy-layout" DeviceIDForHardlinks FlagName = "device-id-for-hardlinks" + VirtualDeviceId FlagName = "virtual-device-id" ExplicitS3AnonymousAuth FlagName = "explicit-s3-anonymous-auth" SafeForgetKeepTags FlagName = "safe-forget-keep-tags" S3Restore FlagName = "s3-restore" @@ -20,6 +21,7 @@ func init() { DeprecateLegacyIndex: {Type: Stable, Description: "disable support for index format used by restic 0.1.0. Use `restic repair index` to update the index if necessary."}, DeprecateS3LegacyLayout: {Type: Stable, Description: "disable support for S3 legacy layout used up to restic 0.7.0. Use restic 0.17.3 to migrate if necessary."}, DeviceIDForHardlinks: {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"}, + VirtualDeviceId: {Type: Alpha, Description: "Use a virtual DeviceID, generated in lexographic order during backup"}, ExplicitS3AnonymousAuth: {Type: Stable, Description: "forbid anonymous S3 authentication unless `-o s3.unsafe-anonymous-auth=true` is set"}, SafeForgetKeepTags: {Type: Stable, Description: "prevent deleting all snapshots if the tag passed to `forget --keep-tags tagname` does not exist"}, S3Restore: {Type: Alpha, Description: "restore S3 objects from cold storage classes when `-o s3.enable-restore=true` is set"},