From 71d167a81be7bde05ad97ba12bf5cbb457c3810e Mon Sep 17 00:00:00 2001 From: Anil Kulkarni Date: Fri, 6 Feb 2026 18:28:53 -0800 Subject: [PATCH] Add support for virtual device ids. Instead of using the system-provided device_ids, this commit modifies the device_id such that the first device seen is given an id of 1, with each device getting an incrementing value. Given that backups are enumerated in the same order, this should result in files being given the same device id on subsequent backups, regardless of what the underlying device id is. This is useful for snapshotting file systems which change the device id for each snapshot. The behavior is gated behind the virtual-device-id feature flag, so as not to change the existing behavior unless explicitly desired --- internal/archiver/archiver.go | 51 ++++++---- internal/archiver/archiver_test.go | 104 ++++++++++++++++---- internal/archiver/file_saver.go | 9 +- internal/archiver/file_saver_test.go | 5 +- internal/archiver/virtual_device_id.go | 80 +++++++++++++++ internal/archiver/virtual_device_id_test.go | 86 ++++++++++++++++ internal/feature/registry.go | 2 + 7 files changed, 295 insertions(+), 42 deletions(-) create mode 100644 internal/archiver/virtual_device_id.go create mode 100644 internal/archiver/virtual_device_id_test.go diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 996e79e6a..3abb0b218 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -249,7 +249,7 @@ func (arch *Archiver) trackItem(item string, previous, current *data.Node, s Ite } // nodeFromFileInfo returns the restic node from an os.FileInfo. -func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) { +func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, deviceIdMap deviceIdMapper, ignoreXattrListError bool) (*data.Node, error) { node, err := meta.ToNode(ignoreXattrListError, func(format string, args ...any) { _ = arch.error(filename, fmt.Errorf(format, args...)) }) @@ -269,6 +269,15 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig node.DeviceID = 0 } } + + if feature.Flag.Enabled(feature.VirtualDeviceId) { + virtualDeviceID, ok := deviceIdMap.GetVirtualId(node.DeviceID) + if !ok { + err = errors.Errorf("A virtual device ID for device %v was not found", node.DeviceID) + return node, arch.error(filename, err) + } + node.DeviceID = virtualDeviceID + } // overwrite name to match that within the snapshot node.Name = path.Base(snPath) // do not filter error for nodes of irregular or invalid type @@ -307,10 +316,10 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { // saveDir stores a directory in the repo and returns the node. snPath is the // path within the current snapshot. -func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous data.TreeNodeIterator, complete fileCompleteFunc) (d futureNode, err error) { +func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous data.TreeNodeIterator, deviceIdMap deviceIdMapper, complete fileCompleteFunc) (d futureNode, err error) { debug.Log("%v %v", snPath, dir) - treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta) + treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta, deviceIdMap) if err != nil { return futureNode{}, err } @@ -334,7 +343,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me return futureNode{}, err } snItem := join(snPath, name) - fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode) + fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode, deviceIdMap) // return error early if possible if err != nil { @@ -359,13 +368,13 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me return fn, nil } -func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (node *data.Node, names []string, err error) { +func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File, deviceIdMap deviceIdMapper) (node *data.Node, names []string, err error) { err = meta.MakeReadable() if err != nil { return nil, nil, fmt.Errorf("openfile for readdirnames failed: %w", err) } - node, err = arch.nodeFromFileInfo(snPath, dir, meta, false) + node, err = arch.nodeFromFileInfo(snPath, dir, meta, deviceIdMap, false) if err != nil { return nil, nil, err } @@ -449,7 +458,7 @@ func (arch *Archiver) allBlobsPresent(previous *data.Node) bool { // Errors and completion needs to be handled by the caller. // // snPath is the path within the current snapshot. -func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *data.Node) (fn futureNode, excluded bool, err error) { +func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *data.Node, deviceIdMap deviceIdMapper) (fn futureNode, excluded bool, err error) { start := time.Now() debug.Log("%v target %q, previous %v", snPath, target, previous) @@ -516,7 +525,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous debug.Log("%v hasn't changed, using old list of blobs", target) arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start)) arch.CompleteBlob(previous.Size) - node, err := arch.nodeFromFileInfo(snPath, target, meta, false) + node, err := arch.nodeFromFileInfo(snPath, target, meta, deviceIdMap, false) if err != nil { return futureNode{}, false, err } @@ -584,7 +593,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous return futureNode{}, false, err } - fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree, + fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree, deviceIdMap, func(node *data.Node, stats ItemStats) { arch.trackItem(snItem, previous, node, stats, time.Since(start)) }) @@ -600,7 +609,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous default: debug.Log(" %v other", target) - node, err := arch.nodeFromFileInfo(snPath, target, meta, false) + node, err := arch.nodeFromFileInfo(snPath, target, meta, deviceIdMap, false) if err != nil { return futureNode{}, false, err } @@ -652,7 +661,7 @@ func join(elem ...string) string { // saveTree stores a Tree in the repo, returned is the tree. snPath is the path // within the current snapshot. -func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous data.TreeNodeIterator, complete fileCompleteFunc) (futureNode, int, error) { +func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous data.TreeNodeIterator, deviceIdMap deviceIdMapper, complete fileCompleteFunc) (futureNode, int, error) { var node *data.Node if snPath != "/" { @@ -661,7 +670,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, } var err error - node, err = arch.dirPathToNode(snPath, atree.FileInfoPath) + node, err = arch.dirPathToNode(snPath, atree.FileInfoPath, deviceIdMap) if err != nil { return futureNode{}, 0, err } @@ -694,7 +703,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, if err != nil { return futureNode{}, 0, err } - fn, excluded, err := arch.save(ctx, pathname, subatree.Path, oldNode) + fn, excluded, err := arch.save(ctx, pathname, subatree.Path, oldNode, deviceIdMap) if err != nil { err = arch.error(subatree.Path, err) @@ -728,7 +737,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, } // not a leaf node, archive subtree - fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *data.Node, is ItemStats) { + fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, deviceIdMap, func(n *data.Node, is ItemStats) { arch.trackItem(snItem, oldNode, n, is, time.Since(start)) }) if err != nil { @@ -747,7 +756,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, return fn, len(nodes), nil } -func (arch *Archiver) dirPathToNode(snPath, target string) (node *data.Node, err error) { +func (arch *Archiver) dirPathToNode(snPath, target string, deviceIdMap deviceIdMapper) (node *data.Node, err error) { meta, err := arch.FS.OpenFile(target, 0, true) if err != nil { return nil, err @@ -762,7 +771,7 @@ func (arch *Archiver) dirPathToNode(snPath, target string) (node *data.Node, err debug.Log("%v, reading dir node data from %v", snPath, target) // in some cases reading xattrs for directories above the backup source is not allowed // thus ignore errors for such folders. - node, err = arch.nodeFromFileInfo(snPath, target, meta, true) + node, err = arch.nodeFromFileInfo(snPath, target, meta, deviceIdMap, true) if err != nil { return nil, err } @@ -842,11 +851,12 @@ func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) dat } // runWorkers starts the worker pools, which are stopped when the context is cancelled. -func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync) { +func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, readOnlyMapper deviceIdMapper) { arch.fileSaver = newFileSaver(ctx, wg, uploader, arch.Repo.Config().ChunkerPolynomial, - arch.Options.ReadConcurrency) + arch.Options.ReadConcurrency, + readOnlyMapper) arch.fileSaver.CompleteBlob = arch.CompleteBlob arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo @@ -881,12 +891,13 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps err = arch.Repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, wgCtx := errgroup.WithContext(ctx) start := time.Now() + deviceIdMap := newMutableDeviceIdMapper() wg.Go(func() error { - arch.runWorkers(wgCtx, wg, uploader) + arch.runWorkers(wgCtx, wg, uploader, deviceIdMap.ReadOnlyMapper()) debug.Log("starting snapshot") - fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *data.Node, is ItemStats) { + fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), deviceIdMap, func(_ *data.Node, is ItemStats) { arch.trackItem("/", nil, nil, is, time.Since(start)) }) if err != nil { diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index b13734655..7175a6e33 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -58,7 +58,8 @@ func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) completeReading := func() { completeReadingCallback = true @@ -221,9 +222,10 @@ func TestArchiverSave(t *testing.T) { var fnr futureNodeResult err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) - node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil) + node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil, deviceIdMap) if err != nil { t.Fatal(err) } @@ -298,9 +300,10 @@ func TestArchiverSaveReaderFS(t *testing.T) { var fnr futureNodeResult err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) - node, excluded, err := arch.save(ctx, "/", filename, nil) + node, excluded, err := arch.save(ctx, "/", filename, nil, deviceIdMap) t.Logf("Save returned %v %v", node, err) if err != nil { t.Fatal(err) @@ -852,10 +855,11 @@ func TestArchiverSaveDir(t *testing.T) { var treeID restic.ID err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) meta, err := testFS.OpenFile(test.target, fs.O_NOFOLLOW, true) rtest.OK(t, err) - ft, err := arch.saveDir(ctx, "/", test.target, meta, nil, nil) + ft, err := arch.saveDir(ctx, "/", test.target, meta, nil, deviceIdMap, nil) rtest.OK(t, err) rtest.OK(t, meta.Close()) @@ -914,10 +918,11 @@ func TestArchiverSaveDirIncremental(t *testing.T) { var fnr futureNodeResult err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) meta, err := testFS.OpenFile(tempdir, fs.O_NOFOLLOW, true) rtest.OK(t, err) - ft, err := arch.saveDir(ctx, "/", tempdir, meta, nil, nil) + ft, err := arch.saveDir(ctx, "/", tempdir, meta, nil, deviceIdMap, nil) rtest.OK(t, err) rtest.OK(t, meta.Close()) @@ -1104,14 +1109,15 @@ func TestArchiverSaveTree(t *testing.T) { var treeID restic.ID err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) atree, err := newTree(testFS, test.targets) if err != nil { t.Fatal(err) } - fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil) + fn, _, err := arch.saveTree(ctx, "/", atree, nil, deviceIdMap, nil) if err != nil { t.Fatal(err) } @@ -2453,10 +2459,11 @@ func TestRacyFileTypeSwap(t *testing.T) { t.Logf("archiver error as expected for %v: %v", item, err) return err } - arch.runWorkers(ctx, wg, uploader) + deviceIdMap := newMutableDeviceIdMapper() + arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper()) // fs.Track will panic if the file was not closed - _, excluded, err := arch.save(ctx, "/", tempfile, nil) + _, excluded, err := arch.save(ctx, "/", tempfile, nil, deviceIdMap) rtest.Assert(t, err != nil && strings.Contains(err.Error(), "changed type, refusing to archive"), "save() returned wrong error: %v", err) tpe := "file" if dirError { @@ -2499,7 +2506,8 @@ func TestMetadataBackupErrorFiltering(t *testing.T) { } // check that errors from reading extended metadata are properly filtered - node, err := arch.nodeFromFileInfo("file", filename+"invalid", nonExistNoder, false) + deviceIdMap := newMutableDeviceIdMapper() + node, err := arch.nodeFromFileInfo("file", filename+"invalid", nonExistNoder, deviceIdMap, false) rtest.Assert(t, node != nil, "node is missing") rtest.Assert(t, err == replacementErr, "expected %v got %v", replacementErr, err) rtest.Assert(t, filteredErr != nil, "missing inner error") @@ -2510,12 +2518,72 @@ func TestMetadataBackupErrorFiltering(t *testing.T) { node: &data.Node{Type: data.NodeTypeIrregular}, err: fmt.Errorf(`unsupported file type "irregular"`), } - node, err = arch.nodeFromFileInfo("file", filename, nonExistNoder, false) + node, err = arch.nodeFromFileInfo("file", filename, nonExistNoder, deviceIdMap, false) rtest.Assert(t, node != nil, "node is missing") rtest.Assert(t, filteredErr == nil, "error for irregular node should not have been filtered") rtest.Assert(t, strings.Contains(err.Error(), "irregular"), "unexpected error %q does not warn about irregular file mode", err) } +func TestVirtualDeviceIdOnlyAffectsHardlinks(t *testing.T) { + // This tests the scenario where both DeviceIdForHardlinks and VirtualDeviceId are set + // In this scenario, due to DeviceIdForHardlinks, all device ID's should be zero, except for hardlinks + // Then, due to VirtualDeviceId, hardlink device ids should be replaced with virtual ids + defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)() + defer feature.TestSetFlag(t, feature.Flag, feature.VirtualDeviceId, true)() + + repo := repository.TestRepository(t) + arch := New(repo, fs.Local{}, Options{}) + deviceIdMap := newMutableDeviceIdMapper() + + normalFile := &mockToNoder{ + node: &data.Node{Type: data.NodeTypeFile, Links: 1, DeviceID: 42}, + err: nil, + } + node, err := arch.nodeFromFileInfo("f", "f", normalFile, deviceIdMap, false) + rtest.OK(t, err) + if node.DeviceID != 0 { + t.Errorf("normal file (Links=1) with both flags: got DeviceID %d, want 0", node.DeviceID) + } + + hardlink := &mockToNoder{ + node: &data.Node{Type: data.NodeTypeFile, Links: 2, DeviceID: 123}, + err: nil, + } + node, err = arch.nodeFromFileInfo("g", "g", hardlink, deviceIdMap, false) + rtest.OK(t, err) + if node.DeviceID != 1 { + t.Errorf("hardlink (Links>1) with both flags: got DeviceID %d, want 1", node.DeviceID) + } +} + +func TestVirtualDeviceId(t *testing.T) { + defer feature.TestSetFlag(t, feature.Flag, feature.VirtualDeviceId, true)() + + repo := repository.TestRepository(t) + arch := New(repo, fs.Local{}, Options{}) + deviceIdMap := newMutableDeviceIdMapper() + + fileDev100 := &mockToNoder{ + node: &data.Node{Type: data.NodeTypeFile, Links: 1, DeviceID: 100}, + err: nil, + } + node1, err := arch.nodeFromFileInfo("a", "a", fileDev100, deviceIdMap, false) + rtest.OK(t, err) + if node1.DeviceID != 1 { + t.Errorf("first file (device 100): got DeviceID %d, want 1", node1.DeviceID) + } + + fileDev200 := &mockToNoder{ + node: &data.Node{Type: data.NodeTypeFile, Links: 1, DeviceID: 200}, + err: nil, + } + node2, err := arch.nodeFromFileInfo("b", "b", fileDev200, deviceIdMap, false) + rtest.OK(t, err) + if node2.DeviceID != 2 { + t.Errorf("second file (device 200): got DeviceID %d, want 2", node2.DeviceID) + } +} + func TestIrregularFile(t *testing.T) { files := TestDir{ "testfile": TestFile{ @@ -2545,7 +2613,8 @@ func TestIrregularFile(t *testing.T) { defer cancel() arch := New(repo, fs.Track{FS: override}, Options{}) - _, excluded, err := arch.save(ctx, "/", tempfile, nil) + deviceIdMap := newMutableDeviceIdMapper() + _, excluded, err := arch.save(ctx, "/", tempfile, nil, deviceIdMap) if err == nil { t.Fatalf("Save() should have failed") } @@ -2595,7 +2664,8 @@ func TestDisappearedFile(t *testing.T) { // the subsequent file.Stat() call. Thus test both cases. for _, errorOnOpen := range []bool{false, true} { arch := New(repo, fs.Track{FS: &missingFS{FS: &fs.Local{}, errorOnOpen: errorOnOpen}}, Options{}) - _, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "testdir"), nil) + deviceIdMap := newMutableDeviceIdMapper() + _, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "testdir"), nil, deviceIdMap) rtest.OK(t, err) rtest.Assert(t, excluded, "testfile should have been excluded") } diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 84e175d82..526da8fcd 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -24,14 +24,16 @@ type fileSaver struct { ch chan<- saveFileJob + deviceIdMap deviceIdMapper + CompleteBlob func(bytes uint64) - NodeFromFileInfo func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) + NodeFromFileInfo func(snPath, filename string, meta ToNoder, deviceIdMap deviceIdMapper, ignoreXattrListError bool) (*data.Node, error) } // newFileSaver returns a new file saver. A worker pool with fileWorkers is // started, it is stopped when ctx is cancelled. -func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, pol chunker.Pol, fileWorkers uint) *fileSaver { +func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, pol chunker.Pol, fileWorkers uint, readOnlyMapper deviceIdMapper) *fileSaver { ch := make(chan saveFileJob) debug.Log("new file saver with %v file workers", fileWorkers) @@ -40,6 +42,7 @@ func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobS saveFilePool: newBufferPool(chunker.MaxSize), pol: pol, ch: ch, + deviceIdMap: readOnlyMapper, CompleteBlob: func(uint64) {}, } @@ -148,7 +151,7 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat debug.Log("%v", snPath) - node, err := s.NodeFromFileInfo(snPath, target, f, false) + node, err := s.NodeFromFileInfo(snPath, target, f, s.deviceIdMap, false) if err != nil { _ = f.Close() completeError(err) diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index 4dbf78548..44303a326 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -40,8 +40,9 @@ func startFileSaver(ctx context.Context, t testing.TB, _ fs.FS) (*fileSaver, *mo } saver := &mockSaver{saved: make(map[string]int)} - s := newFileSaver(ctx, wg, saver, pol, workers) - s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) { + deviceIdMap := newMutableDeviceIdMapper().ReadOnlyMapper() + s := newFileSaver(ctx, wg, saver, pol, workers, deviceIdMap) + s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, deviceIdMap deviceIdMapper, ignoreXattrListError bool) (*data.Node, error) { return meta.ToNode(ignoreXattrListError, t.Logf) } diff --git a/internal/archiver/virtual_device_id.go b/internal/archiver/virtual_device_id.go new file mode 100644 index 000000000..b8aff9510 --- /dev/null +++ b/internal/archiver/virtual_device_id.go @@ -0,0 +1,80 @@ +package archiver + +// Virtual device ID mapping: converts real device IDs into a stable virtual +// numbering 1..n, where assignment order is determined by first-seen. This +// keeps snapshot trees stable across runs regardless of which devices appear +// or in what order. +// +// Concurrency: only the goroutine that created the mapper via +// newMutableDeviceIdMapper() may call GetVirtualId on that mapper. For that +// goroutine, GetVirtualId may allocate a new virtual ID for an unseen device. +// All other goroutines must use the result of ReadOnlyMapper(). On the +// read-only mapper, GetVirtualId returns (0, false) for any device ID not yet +// seen by the mutable owner. +// +// See deviceIdMapper and newMutableDeviceIdMapper. + +import ( + "sync" + + "github.com/restic/restic/internal/debug" +) + +// deviceIdMapper maps real device IDs to stable virtual IDs. Implementations +// are either mutable (may assign new virtual IDs) or read-only (lookup only). +type deviceIdMapper interface { + GetVirtualId(realDeviceID uint64) (uint64, bool) + ReadOnlyMapper() deviceIdMapper +} + +// newMutableDeviceIdMapper returns a mapper that may assign new virtual IDs. +// Only the creating goroutine may call GetVirtualId on the returned value. +func newMutableDeviceIdMapper() deviceIdMapper { + m := &deviceIdMap{ + count: 0, + cache: &vIdCache{}, + } + m.cache.set(0, 0) + return m +} + +// vIdCache is a concurrent read-only view of realID -> virtualID; GetVirtualId returns (0, false) when missing. +type vIdCache sync.Map + +func (v *vIdCache) GetVirtualId(realDeviceID uint64) (uint64, bool) { + if id, ok := (*sync.Map)(v).Load(realDeviceID); ok { + return id.(uint64), true + } + return 0, false +} + +func (v *vIdCache) set(realDeviceID uint64, virtualDeviceID uint64) { + (*sync.Map)(v).Store(realDeviceID, virtualDeviceID) +} + +func (v *vIdCache) ReadOnlyMapper() deviceIdMapper { + return v +} + +// deviceIdMap is the mutable mapper; only its owner goroutine may call GetVirtualId. +type deviceIdMap struct { + count uint64 + cache *vIdCache +} + +// GetVirtualId returns the virtual ID for realDeviceID, allocating one (first-seen order) if needed. +func (d *deviceIdMap) GetVirtualId(realDeviceID uint64) (uint64, bool) { + id, ok := d.cache.GetVirtualId(realDeviceID) + if !ok { + id = d.count + 1 + d.cache.set(realDeviceID, id) + d.count = id + debug.Log("Mapped deviceId %v to virtualId %v", realDeviceID, id) + } + return id, true +} + +// ReadOnlyMapper returns a concurrent-safe view for other goroutines; they must use this, not GetVirtualId on d. +func (d *deviceIdMap) ReadOnlyMapper() deviceIdMapper { + return d.cache +} diff --git a/internal/archiver/virtual_device_id_test.go b/internal/archiver/virtual_device_id_test.go new file mode 100644 index 000000000..1bd51be3f --- /dev/null +++ b/internal/archiver/virtual_device_id_test.go @@ -0,0 +1,86 @@ +package archiver + +import ( + "sync" + "testing" +) + +func TestVirtualIdFirstSeenOrder(t *testing.T) { + m := newMutableDeviceIdMapper() + + // Real 0 is pre-mapped to virtual 0 + if vid, ok := m.GetVirtualId(0); !ok || vid != 0 { + t.Fatalf("real 0: got (%d, %v), want (0, true)", vid, ok) + } + + // First-seen order: 100 -> 1, 200 -> 2, 100 again -> 1, 300 -> 3 + cases := []struct { + realID uint64 + want uint64 + }{ + {100, 1}, + {200, 2}, + {100, 1}, + {300, 3}, + } + for _, c := range cases { + vid, ok := m.GetVirtualId(c.realID) + if !ok || vid != c.want { + t.Errorf("real %d: got (%d, %v), want (%d, true)", c.realID, vid, ok, c.want) + } + } +} + +func TestReadOnlyMapperUnseenReturnsFalse(t *testing.T) { + m := newMutableDeviceIdMapper() + m.GetVirtualId(42) + ro := m.ReadOnlyMapper() + + if vid, ok := ro.GetVirtualId(999); ok || vid != 0 { + t.Errorf("unseen 999: got (%d, %v), want (0, false)", vid, ok) + } +} + +func TestReadOnlyMapperSeenReturnsId(t *testing.T) { + m := newMutableDeviceIdMapper() + vidOwner, _ := m.GetVirtualId(100) + ro := m.ReadOnlyMapper() + + vidRo, ok := ro.GetVirtualId(100) + if !ok || vidRo != vidOwner { + t.Errorf("read-only for seen 100: got (%d, %v), want (%d, true)", vidRo, ok, vidOwner) + } +} + +func TestReadOnlyMapperConcurrent(t *testing.T) { + m := newMutableDeviceIdMapper() + ro := m.ReadOnlyMapper() + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = ro.GetVirtualId(1) + _, _ = ro.GetVirtualId(999) + }() + } + + m.GetVirtualId(1) + wg.Wait() +} + +func TestReadOnlyMapperSeesUpdatesFromOwner(t *testing.T) { + m := newMutableDeviceIdMapper() + ro := m.ReadOnlyMapper() + + if _, ok := ro.GetVirtualId(1); ok { + t.Error("read-only saw 1 before owner") + } + + m.GetVirtualId(1) + vid, ok := ro.GetVirtualId(1) + if !ok || vid != 1 { + t.Errorf("after owner added 1: got (%d, %v), want (1, true)", vid, ok) + } +} diff --git a/internal/feature/registry.go b/internal/feature/registry.go index a7368fa75..0d68a9f75 100644 --- a/internal/feature/registry.go +++ b/internal/feature/registry.go @@ -9,6 +9,7 @@ const ( DeprecateLegacyIndex FlagName = "deprecate-legacy-index" DeprecateS3LegacyLayout FlagName = "deprecate-s3-legacy-layout" DeviceIDForHardlinks FlagName = "device-id-for-hardlinks" + VirtualDeviceId FlagName = "virtual-device-id" ExplicitS3AnonymousAuth FlagName = "explicit-s3-anonymous-auth" SafeForgetKeepTags FlagName = "safe-forget-keep-tags" S3Restore FlagName = "s3-restore" @@ -20,6 +21,7 @@ func init() { DeprecateLegacyIndex: {Type: Stable, Description: "disable support for index format used by restic 0.1.0. Use `restic repair index` to update the index if necessary."}, DeprecateS3LegacyLayout: {Type: Stable, Description: "disable support for S3 legacy layout used up to restic 0.7.0. Use restic 0.17.3 to migrate if necessary."}, DeviceIDForHardlinks: {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"}, + VirtualDeviceId: {Type: Alpha, Description: "Use a virtual DeviceID, generated in lexographic order during backup"}, ExplicitS3AnonymousAuth: {Type: Stable, Description: "forbid anonymous S3 authentication unless `-o s3.unsafe-anonymous-auth=true` is set"}, SafeForgetKeepTags: {Type: Stable, Description: "prevent deleting all snapshots if the tag passed to `forget --keep-tags tagname` does not exist"}, S3Restore: {Type: Alpha, Description: "restore S3 objects from cold storage classes when `-o s3.enable-restore=true` is set"},