mirror of
https://github.com/restic/restic.git
synced 2026-05-28 04:35:41 -04:00
Add support for virtual device ids. Instead of using the system-provided
device_ids, this commit modifies the device_id such that the first device seen is given an id of 1, with each device getting an incrementing value. Given that backups are enumerated in the same order, this should result in files being given the same device id on subsequent backups, regardless of what the underlying device id is. This is useful for snapshotting file systems which change the device id for each snapshot. The behavior is gated behind the virtual-device-id feature flag, so as not to change the existing behavior unless explicitly desired
This commit is contained in:
Anil Kulkarni
parent
7101f11133
commit
71d167a81b
7 changed files with 295 additions and 42 deletions
|
|
@ -249,7 +249,7 @@ func (arch *Archiver) trackItem(item string, previous, current *data.Node, s Ite
|
|||
}
|
||||
|
||||
// nodeFromFileInfo returns the restic node from an os.FileInfo.
|
||||
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) {
|
||||
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, deviceIdMap deviceIdMapper, ignoreXattrListError bool) (*data.Node, error) {
|
||||
node, err := meta.ToNode(ignoreXattrListError, func(format string, args ...any) {
|
||||
_ = arch.error(filename, fmt.Errorf(format, args...))
|
||||
})
|
||||
|
|
@ -269,6 +269,15 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig
|
|||
node.DeviceID = 0
|
||||
}
|
||||
}
|
||||
|
||||
if feature.Flag.Enabled(feature.VirtualDeviceId) {
|
||||
virtualDeviceID, ok := deviceIdMap.GetVirtualId(node.DeviceID)
|
||||
if !ok {
|
||||
err = errors.Errorf("A virtual device ID for device %v was not found", node.DeviceID)
|
||||
return node, arch.error(filename, err)
|
||||
}
|
||||
node.DeviceID = virtualDeviceID
|
||||
}
|
||||
// overwrite name to match that within the snapshot
|
||||
node.Name = path.Base(snPath)
|
||||
// do not filter error for nodes of irregular or invalid type
|
||||
|
|
@ -307,10 +316,10 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
|
|||
|
||||
// saveDir stores a directory in the repo and returns the node. snPath is the
|
||||
// path within the current snapshot.
|
||||
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous data.TreeNodeIterator, complete fileCompleteFunc) (d futureNode, err error) {
|
||||
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous data.TreeNodeIterator, deviceIdMap deviceIdMapper, complete fileCompleteFunc) (d futureNode, err error) {
|
||||
debug.Log("%v %v", snPath, dir)
|
||||
|
||||
treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta)
|
||||
treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta, deviceIdMap)
|
||||
if err != nil {
|
||||
return futureNode{}, err
|
||||
}
|
||||
|
|
@ -334,7 +343,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
|
|||
return futureNode{}, err
|
||||
}
|
||||
snItem := join(snPath, name)
|
||||
fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode)
|
||||
fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode, deviceIdMap)
|
||||
|
||||
// return error early if possible
|
||||
if err != nil {
|
||||
|
|
@ -359,13 +368,13 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
|
|||
return fn, nil
|
||||
}
|
||||
|
||||
func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (node *data.Node, names []string, err error) {
|
||||
func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File, deviceIdMap deviceIdMapper) (node *data.Node, names []string, err error) {
|
||||
err = meta.MakeReadable()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("openfile for readdirnames failed: %w", err)
|
||||
}
|
||||
|
||||
node, err = arch.nodeFromFileInfo(snPath, dir, meta, false)
|
||||
node, err = arch.nodeFromFileInfo(snPath, dir, meta, deviceIdMap, false)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
|
@ -449,7 +458,7 @@ func (arch *Archiver) allBlobsPresent(previous *data.Node) bool {
|
|||
// Errors and completion needs to be handled by the caller.
|
||||
//
|
||||
// snPath is the path within the current snapshot.
|
||||
func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *data.Node) (fn futureNode, excluded bool, err error) {
|
||||
func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *data.Node, deviceIdMap deviceIdMapper) (fn futureNode, excluded bool, err error) {
|
||||
start := time.Now()
|
||||
|
||||
debug.Log("%v target %q, previous %v", snPath, target, previous)
|
||||
|
|
@ -516,7 +525,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
|
|||
debug.Log("%v hasn't changed, using old list of blobs", target)
|
||||
arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
|
||||
arch.CompleteBlob(previous.Size)
|
||||
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
|
||||
node, err := arch.nodeFromFileInfo(snPath, target, meta, deviceIdMap, false)
|
||||
if err != nil {
|
||||
return futureNode{}, false, err
|
||||
}
|
||||
|
|
@ -584,7 +593,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
|
|||
return futureNode{}, false, err
|
||||
}
|
||||
|
||||
fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree,
|
||||
fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree, deviceIdMap,
|
||||
func(node *data.Node, stats ItemStats) {
|
||||
arch.trackItem(snItem, previous, node, stats, time.Since(start))
|
||||
})
|
||||
|
|
@ -600,7 +609,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
|
|||
default:
|
||||
debug.Log(" %v other", target)
|
||||
|
||||
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
|
||||
node, err := arch.nodeFromFileInfo(snPath, target, meta, deviceIdMap, false)
|
||||
if err != nil {
|
||||
return futureNode{}, false, err
|
||||
}
|
||||
|
|
@ -652,7 +661,7 @@ func join(elem ...string) string {
|
|||
|
||||
// saveTree stores a Tree in the repo, returned is the tree. snPath is the path
|
||||
// within the current snapshot.
|
||||
func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous data.TreeNodeIterator, complete fileCompleteFunc) (futureNode, int, error) {
|
||||
func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous data.TreeNodeIterator, deviceIdMap deviceIdMapper, complete fileCompleteFunc) (futureNode, int, error) {
|
||||
|
||||
var node *data.Node
|
||||
if snPath != "/" {
|
||||
|
|
@ -661,7 +670,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
|
|||
}
|
||||
|
||||
var err error
|
||||
node, err = arch.dirPathToNode(snPath, atree.FileInfoPath)
|
||||
node, err = arch.dirPathToNode(snPath, atree.FileInfoPath, deviceIdMap)
|
||||
if err != nil {
|
||||
return futureNode{}, 0, err
|
||||
}
|
||||
|
|
@ -694,7 +703,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
|
|||
if err != nil {
|
||||
return futureNode{}, 0, err
|
||||
}
|
||||
fn, excluded, err := arch.save(ctx, pathname, subatree.Path, oldNode)
|
||||
fn, excluded, err := arch.save(ctx, pathname, subatree.Path, oldNode, deviceIdMap)
|
||||
|
||||
if err != nil {
|
||||
err = arch.error(subatree.Path, err)
|
||||
|
|
@ -728,7 +737,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
|
|||
}
|
||||
|
||||
// not a leaf node, archive subtree
|
||||
fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *data.Node, is ItemStats) {
|
||||
fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, deviceIdMap, func(n *data.Node, is ItemStats) {
|
||||
arch.trackItem(snItem, oldNode, n, is, time.Since(start))
|
||||
})
|
||||
if err != nil {
|
||||
|
|
@ -747,7 +756,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
|
|||
return fn, len(nodes), nil
|
||||
}
|
||||
|
||||
func (arch *Archiver) dirPathToNode(snPath, target string) (node *data.Node, err error) {
|
||||
func (arch *Archiver) dirPathToNode(snPath, target string, deviceIdMap deviceIdMapper) (node *data.Node, err error) {
|
||||
meta, err := arch.FS.OpenFile(target, 0, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -762,7 +771,7 @@ func (arch *Archiver) dirPathToNode(snPath, target string) (node *data.Node, err
|
|||
debug.Log("%v, reading dir node data from %v", snPath, target)
|
||||
// in some cases reading xattrs for directories above the backup source is not allowed
|
||||
// thus ignore errors for such folders.
|
||||
node, err = arch.nodeFromFileInfo(snPath, target, meta, true)
|
||||
node, err = arch.nodeFromFileInfo(snPath, target, meta, deviceIdMap, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -842,11 +851,12 @@ func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) dat
|
|||
}
|
||||
|
||||
// runWorkers starts the worker pools, which are stopped when the context is cancelled.
|
||||
func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync) {
|
||||
func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, readOnlyMapper deviceIdMapper) {
|
||||
arch.fileSaver = newFileSaver(ctx, wg,
|
||||
uploader,
|
||||
arch.Repo.Config().ChunkerPolynomial,
|
||||
arch.Options.ReadConcurrency)
|
||||
arch.Options.ReadConcurrency,
|
||||
readOnlyMapper)
|
||||
arch.fileSaver.CompleteBlob = arch.CompleteBlob
|
||||
arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo
|
||||
|
||||
|
|
@ -881,12 +891,13 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
|
|||
err = arch.Repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
wg, wgCtx := errgroup.WithContext(ctx)
|
||||
start := time.Now()
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
|
||||
wg.Go(func() error {
|
||||
arch.runWorkers(wgCtx, wg, uploader)
|
||||
arch.runWorkers(wgCtx, wg, uploader, deviceIdMap.ReadOnlyMapper())
|
||||
|
||||
debug.Log("starting snapshot")
|
||||
fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *data.Node, is ItemStats) {
|
||||
fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), deviceIdMap, func(_ *data.Node, is ItemStats) {
|
||||
arch.trackItem("/", nil, nil, is, time.Since(start))
|
||||
})
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -58,7 +58,8 @@ func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS
|
|||
|
||||
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
arch.runWorkers(ctx, wg, uploader)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper())
|
||||
|
||||
completeReading := func() {
|
||||
completeReadingCallback = true
|
||||
|
|
@ -221,9 +222,10 @@ func TestArchiverSave(t *testing.T) {
|
|||
var fnr futureNodeResult
|
||||
err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
arch.runWorkers(ctx, wg, uploader)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper())
|
||||
|
||||
node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil)
|
||||
node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil, deviceIdMap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -298,9 +300,10 @@ func TestArchiverSaveReaderFS(t *testing.T) {
|
|||
var fnr futureNodeResult
|
||||
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
arch.runWorkers(ctx, wg, uploader)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper())
|
||||
|
||||
node, excluded, err := arch.save(ctx, "/", filename, nil)
|
||||
node, excluded, err := arch.save(ctx, "/", filename, nil, deviceIdMap)
|
||||
t.Logf("Save returned %v %v", node, err)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -852,10 +855,11 @@ func TestArchiverSaveDir(t *testing.T) {
|
|||
var treeID restic.ID
|
||||
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
arch.runWorkers(ctx, wg, uploader)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper())
|
||||
meta, err := testFS.OpenFile(test.target, fs.O_NOFOLLOW, true)
|
||||
rtest.OK(t, err)
|
||||
ft, err := arch.saveDir(ctx, "/", test.target, meta, nil, nil)
|
||||
ft, err := arch.saveDir(ctx, "/", test.target, meta, nil, deviceIdMap, nil)
|
||||
rtest.OK(t, err)
|
||||
rtest.OK(t, meta.Close())
|
||||
|
||||
|
|
@ -914,10 +918,11 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
|
|||
var fnr futureNodeResult
|
||||
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
arch.runWorkers(ctx, wg, uploader)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper())
|
||||
meta, err := testFS.OpenFile(tempdir, fs.O_NOFOLLOW, true)
|
||||
rtest.OK(t, err)
|
||||
ft, err := arch.saveDir(ctx, "/", tempdir, meta, nil, nil)
|
||||
ft, err := arch.saveDir(ctx, "/", tempdir, meta, nil, deviceIdMap, nil)
|
||||
rtest.OK(t, err)
|
||||
rtest.OK(t, meta.Close())
|
||||
|
||||
|
|
@ -1104,14 +1109,15 @@ func TestArchiverSaveTree(t *testing.T) {
|
|||
var treeID restic.ID
|
||||
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
arch.runWorkers(ctx, wg, uploader)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper())
|
||||
|
||||
atree, err := newTree(testFS, test.targets)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil)
|
||||
fn, _, err := arch.saveTree(ctx, "/", atree, nil, deviceIdMap, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -2453,10 +2459,11 @@ func TestRacyFileTypeSwap(t *testing.T) {
|
|||
t.Logf("archiver error as expected for %v: %v", item, err)
|
||||
return err
|
||||
}
|
||||
arch.runWorkers(ctx, wg, uploader)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
arch.runWorkers(ctx, wg, uploader, deviceIdMap.ReadOnlyMapper())
|
||||
|
||||
// fs.Track will panic if the file was not closed
|
||||
_, excluded, err := arch.save(ctx, "/", tempfile, nil)
|
||||
_, excluded, err := arch.save(ctx, "/", tempfile, nil, deviceIdMap)
|
||||
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "changed type, refusing to archive"), "save() returned wrong error: %v", err)
|
||||
tpe := "file"
|
||||
if dirError {
|
||||
|
|
@ -2499,7 +2506,8 @@ func TestMetadataBackupErrorFiltering(t *testing.T) {
|
|||
}
|
||||
|
||||
// check that errors from reading extended metadata are properly filtered
|
||||
node, err := arch.nodeFromFileInfo("file", filename+"invalid", nonExistNoder, false)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
node, err := arch.nodeFromFileInfo("file", filename+"invalid", nonExistNoder, deviceIdMap, false)
|
||||
rtest.Assert(t, node != nil, "node is missing")
|
||||
rtest.Assert(t, err == replacementErr, "expected %v got %v", replacementErr, err)
|
||||
rtest.Assert(t, filteredErr != nil, "missing inner error")
|
||||
|
|
@ -2510,12 +2518,72 @@ func TestMetadataBackupErrorFiltering(t *testing.T) {
|
|||
node: &data.Node{Type: data.NodeTypeIrregular},
|
||||
err: fmt.Errorf(`unsupported file type "irregular"`),
|
||||
}
|
||||
node, err = arch.nodeFromFileInfo("file", filename, nonExistNoder, false)
|
||||
node, err = arch.nodeFromFileInfo("file", filename, nonExistNoder, deviceIdMap, false)
|
||||
rtest.Assert(t, node != nil, "node is missing")
|
||||
rtest.Assert(t, filteredErr == nil, "error for irregular node should not have been filtered")
|
||||
rtest.Assert(t, strings.Contains(err.Error(), "irregular"), "unexpected error %q does not warn about irregular file mode", err)
|
||||
}
|
||||
|
||||
func TestVirtualDeviceIdOnlyAffectsHardlinks(t *testing.T) {
|
||||
// This tests the scenario where both DeviceIdForHardlinks and VirtualDeviceId are set
|
||||
// In this scenario, due to DeviceIdForHardlinks, all device ID's should be zero, except for hardlinks
|
||||
// Then, due to VirtualDeviceId, hardlink device ids should be replaced with virtual ids
|
||||
defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)()
|
||||
defer feature.TestSetFlag(t, feature.Flag, feature.VirtualDeviceId, true)()
|
||||
|
||||
repo := repository.TestRepository(t)
|
||||
arch := New(repo, fs.Local{}, Options{})
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
|
||||
normalFile := &mockToNoder{
|
||||
node: &data.Node{Type: data.NodeTypeFile, Links: 1, DeviceID: 42},
|
||||
err: nil,
|
||||
}
|
||||
node, err := arch.nodeFromFileInfo("f", "f", normalFile, deviceIdMap, false)
|
||||
rtest.OK(t, err)
|
||||
if node.DeviceID != 0 {
|
||||
t.Errorf("normal file (Links=1) with both flags: got DeviceID %d, want 0", node.DeviceID)
|
||||
}
|
||||
|
||||
hardlink := &mockToNoder{
|
||||
node: &data.Node{Type: data.NodeTypeFile, Links: 2, DeviceID: 123},
|
||||
err: nil,
|
||||
}
|
||||
node, err = arch.nodeFromFileInfo("g", "g", hardlink, deviceIdMap, false)
|
||||
rtest.OK(t, err)
|
||||
if node.DeviceID != 1 {
|
||||
t.Errorf("hardlink (Links>1) with both flags: got DeviceID %d, want 1", node.DeviceID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualDeviceId(t *testing.T) {
|
||||
defer feature.TestSetFlag(t, feature.Flag, feature.VirtualDeviceId, true)()
|
||||
|
||||
repo := repository.TestRepository(t)
|
||||
arch := New(repo, fs.Local{}, Options{})
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
|
||||
fileDev100 := &mockToNoder{
|
||||
node: &data.Node{Type: data.NodeTypeFile, Links: 1, DeviceID: 100},
|
||||
err: nil,
|
||||
}
|
||||
node1, err := arch.nodeFromFileInfo("a", "a", fileDev100, deviceIdMap, false)
|
||||
rtest.OK(t, err)
|
||||
if node1.DeviceID != 1 {
|
||||
t.Errorf("first file (device 100): got DeviceID %d, want 1", node1.DeviceID)
|
||||
}
|
||||
|
||||
fileDev200 := &mockToNoder{
|
||||
node: &data.Node{Type: data.NodeTypeFile, Links: 1, DeviceID: 200},
|
||||
err: nil,
|
||||
}
|
||||
node2, err := arch.nodeFromFileInfo("b", "b", fileDev200, deviceIdMap, false)
|
||||
rtest.OK(t, err)
|
||||
if node2.DeviceID != 2 {
|
||||
t.Errorf("second file (device 200): got DeviceID %d, want 2", node2.DeviceID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIrregularFile(t *testing.T) {
|
||||
files := TestDir{
|
||||
"testfile": TestFile{
|
||||
|
|
@ -2545,7 +2613,8 @@ func TestIrregularFile(t *testing.T) {
|
|||
defer cancel()
|
||||
|
||||
arch := New(repo, fs.Track{FS: override}, Options{})
|
||||
_, excluded, err := arch.save(ctx, "/", tempfile, nil)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
_, excluded, err := arch.save(ctx, "/", tempfile, nil, deviceIdMap)
|
||||
if err == nil {
|
||||
t.Fatalf("Save() should have failed")
|
||||
}
|
||||
|
|
@ -2595,7 +2664,8 @@ func TestDisappearedFile(t *testing.T) {
|
|||
// the subsequent file.Stat() call. Thus test both cases.
|
||||
for _, errorOnOpen := range []bool{false, true} {
|
||||
arch := New(repo, fs.Track{FS: &missingFS{FS: &fs.Local{}, errorOnOpen: errorOnOpen}}, Options{})
|
||||
_, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "testdir"), nil)
|
||||
deviceIdMap := newMutableDeviceIdMapper()
|
||||
_, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "testdir"), nil, deviceIdMap)
|
||||
rtest.OK(t, err)
|
||||
rtest.Assert(t, excluded, "testfile should have been excluded")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,14 +24,16 @@ type fileSaver struct {
|
|||
|
||||
ch chan<- saveFileJob
|
||||
|
||||
deviceIdMap deviceIdMapper
|
||||
|
||||
CompleteBlob func(bytes uint64)
|
||||
|
||||
NodeFromFileInfo func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error)
|
||||
NodeFromFileInfo func(snPath, filename string, meta ToNoder, deviceIdMap deviceIdMapper, ignoreXattrListError bool) (*data.Node, error)
|
||||
}
|
||||
|
||||
// newFileSaver returns a new file saver. A worker pool with fileWorkers is
|
||||
// started, it is stopped when ctx is cancelled.
|
||||
func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, pol chunker.Pol, fileWorkers uint) *fileSaver {
|
||||
func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, pol chunker.Pol, fileWorkers uint, readOnlyMapper deviceIdMapper) *fileSaver {
|
||||
ch := make(chan saveFileJob)
|
||||
debug.Log("new file saver with %v file workers", fileWorkers)
|
||||
|
||||
|
|
@ -40,6 +42,7 @@ func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobS
|
|||
saveFilePool: newBufferPool(chunker.MaxSize),
|
||||
pol: pol,
|
||||
ch: ch,
|
||||
deviceIdMap: readOnlyMapper,
|
||||
|
||||
CompleteBlob: func(uint64) {},
|
||||
}
|
||||
|
|
@ -148,7 +151,7 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat
|
|||
|
||||
debug.Log("%v", snPath)
|
||||
|
||||
node, err := s.NodeFromFileInfo(snPath, target, f, false)
|
||||
node, err := s.NodeFromFileInfo(snPath, target, f, s.deviceIdMap, false)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
completeError(err)
|
||||
|
|
|
|||
|
|
@ -40,8 +40,9 @@ func startFileSaver(ctx context.Context, t testing.TB, _ fs.FS) (*fileSaver, *mo
|
|||
}
|
||||
|
||||
saver := &mockSaver{saved: make(map[string]int)}
|
||||
s := newFileSaver(ctx, wg, saver, pol, workers)
|
||||
s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) {
|
||||
deviceIdMap := newMutableDeviceIdMapper().ReadOnlyMapper()
|
||||
s := newFileSaver(ctx, wg, saver, pol, workers, deviceIdMap)
|
||||
s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, deviceIdMap deviceIdMapper, ignoreXattrListError bool) (*data.Node, error) {
|
||||
return meta.ToNode(ignoreXattrListError, t.Logf)
|
||||
}
|
||||
|
||||
|
|
|
|||
80
internal/archiver/virtual_device_id.go
Normal file
80
internal/archiver/virtual_device_id.go
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
package archiver
|
||||
|
||||
// Virtual device ID mapping: converts real device IDs into a stable virtual
|
||||
// numbering 1..n, where assignment order is determined by first-seen. This
|
||||
// keeps snapshot trees stable across runs regardless of which devices appear
|
||||
// or in what order.
|
||||
//
|
||||
// Concurrency: only the goroutine that created the mapper via
|
||||
// newMutableDeviceIdMapper() may call GetVirtualId on that mapper. For that
|
||||
// goroutine, GetVirtualId may allocate a new virtual ID for an unseen device.
|
||||
// All other goroutines must use the result of ReadOnlyMapper(). On the
|
||||
// read-only mapper, GetVirtualId returns (0, false) for any device ID not yet
|
||||
// seen by the mutable owner.
|
||||
//
|
||||
// See deviceIdMapper and newMutableDeviceIdMapper.
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
)
|
||||
|
||||
// deviceIdMapper maps real device IDs to stable virtual IDs. Implementations
|
||||
// are either mutable (may assign new virtual IDs) or read-only (lookup only).
|
||||
type deviceIdMapper interface {
|
||||
GetVirtualId(realDeviceID uint64) (uint64, bool)
|
||||
ReadOnlyMapper() deviceIdMapper
|
||||
}
|
||||
|
||||
// newMutableDeviceIdMapper returns a mapper that may assign new virtual IDs.
|
||||
// Only the creating goroutine may call GetVirtualId on the returned value.
|
||||
func newMutableDeviceIdMapper() deviceIdMapper {
|
||||
m := &deviceIdMap{
|
||||
count: 0,
|
||||
cache: &vIdCache{},
|
||||
}
|
||||
m.cache.set(0, 0)
|
||||
return m
|
||||
}
|
||||
|
||||
// vIdCache is a concurrent read-only view of realID -> virtualID; GetVirtualId returns (0, false) when missing.
|
||||
type vIdCache sync.Map
|
||||
|
||||
func (v *vIdCache) GetVirtualId(realDeviceID uint64) (uint64, bool) {
|
||||
if id, ok := (*sync.Map)(v).Load(realDeviceID); ok {
|
||||
return id.(uint64), true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (v *vIdCache) set(realDeviceID uint64, virtualDeviceID uint64) {
|
||||
(*sync.Map)(v).Store(realDeviceID, virtualDeviceID)
|
||||
}
|
||||
|
||||
func (v *vIdCache) ReadOnlyMapper() deviceIdMapper {
|
||||
return v
|
||||
}
|
||||
|
||||
// deviceIdMap is the mutable mapper; only its owner goroutine may call GetVirtualId.
|
||||
type deviceIdMap struct {
|
||||
count uint64
|
||||
cache *vIdCache
|
||||
}
|
||||
|
||||
// GetVirtualId returns the virtual ID for realDeviceID, allocating one (first-seen order) if needed.
|
||||
func (d *deviceIdMap) GetVirtualId(realDeviceID uint64) (uint64, bool) {
|
||||
id, ok := d.cache.GetVirtualId(realDeviceID)
|
||||
if !ok {
|
||||
id = d.count + 1
|
||||
d.cache.set(realDeviceID, id)
|
||||
d.count = id
|
||||
debug.Log("Mapped deviceId %v to virtualId %v", realDeviceID, id)
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
// ReadOnlyMapper returns a concurrent-safe view for other goroutines; they must use this, not GetVirtualId on d.
|
||||
func (d *deviceIdMap) ReadOnlyMapper() deviceIdMapper {
|
||||
return d.cache
|
||||
}
|
||||
86
internal/archiver/virtual_device_id_test.go
Normal file
86
internal/archiver/virtual_device_id_test.go
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
package archiver
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestVirtualIdFirstSeenOrder(t *testing.T) {
|
||||
m := newMutableDeviceIdMapper()
|
||||
|
||||
// Real 0 is pre-mapped to virtual 0
|
||||
if vid, ok := m.GetVirtualId(0); !ok || vid != 0 {
|
||||
t.Fatalf("real 0: got (%d, %v), want (0, true)", vid, ok)
|
||||
}
|
||||
|
||||
// First-seen order: 100 -> 1, 200 -> 2, 100 again -> 1, 300 -> 3
|
||||
cases := []struct {
|
||||
realID uint64
|
||||
want uint64
|
||||
}{
|
||||
{100, 1},
|
||||
{200, 2},
|
||||
{100, 1},
|
||||
{300, 3},
|
||||
}
|
||||
for _, c := range cases {
|
||||
vid, ok := m.GetVirtualId(c.realID)
|
||||
if !ok || vid != c.want {
|
||||
t.Errorf("real %d: got (%d, %v), want (%d, true)", c.realID, vid, ok, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadOnlyMapperUnseenReturnsFalse(t *testing.T) {
|
||||
m := newMutableDeviceIdMapper()
|
||||
m.GetVirtualId(42)
|
||||
ro := m.ReadOnlyMapper()
|
||||
|
||||
if vid, ok := ro.GetVirtualId(999); ok || vid != 0 {
|
||||
t.Errorf("unseen 999: got (%d, %v), want (0, false)", vid, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadOnlyMapperSeenReturnsId(t *testing.T) {
|
||||
m := newMutableDeviceIdMapper()
|
||||
vidOwner, _ := m.GetVirtualId(100)
|
||||
ro := m.ReadOnlyMapper()
|
||||
|
||||
vidRo, ok := ro.GetVirtualId(100)
|
||||
if !ok || vidRo != vidOwner {
|
||||
t.Errorf("read-only for seen 100: got (%d, %v), want (%d, true)", vidRo, ok, vidOwner)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadOnlyMapperConcurrent(t *testing.T) {
|
||||
m := newMutableDeviceIdMapper()
|
||||
ro := m.ReadOnlyMapper()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, _ = ro.GetVirtualId(1)
|
||||
_, _ = ro.GetVirtualId(999)
|
||||
}()
|
||||
}
|
||||
|
||||
m.GetVirtualId(1)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestReadOnlyMapperSeesUpdatesFromOwner(t *testing.T) {
|
||||
m := newMutableDeviceIdMapper()
|
||||
ro := m.ReadOnlyMapper()
|
||||
|
||||
if _, ok := ro.GetVirtualId(1); ok {
|
||||
t.Error("read-only saw 1 before owner")
|
||||
}
|
||||
|
||||
m.GetVirtualId(1)
|
||||
vid, ok := ro.GetVirtualId(1)
|
||||
if !ok || vid != 1 {
|
||||
t.Errorf("after owner added 1: got (%d, %v), want (1, true)", vid, ok)
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ const (
|
|||
DeprecateLegacyIndex FlagName = "deprecate-legacy-index"
|
||||
DeprecateS3LegacyLayout FlagName = "deprecate-s3-legacy-layout"
|
||||
DeviceIDForHardlinks FlagName = "device-id-for-hardlinks"
|
||||
VirtualDeviceId FlagName = "virtual-device-id"
|
||||
ExplicitS3AnonymousAuth FlagName = "explicit-s3-anonymous-auth"
|
||||
SafeForgetKeepTags FlagName = "safe-forget-keep-tags"
|
||||
S3Restore FlagName = "s3-restore"
|
||||
|
|
@ -20,6 +21,7 @@ func init() {
|
|||
DeprecateLegacyIndex: {Type: Stable, Description: "disable support for index format used by restic 0.1.0. Use `restic repair index` to update the index if necessary."},
|
||||
DeprecateS3LegacyLayout: {Type: Stable, Description: "disable support for S3 legacy layout used up to restic 0.7.0. Use restic 0.17.3 to migrate if necessary."},
|
||||
DeviceIDForHardlinks: {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"},
|
||||
VirtualDeviceId: {Type: Alpha, Description: "Use a virtual DeviceID, generated in lexographic order during backup"},
|
||||
ExplicitS3AnonymousAuth: {Type: Stable, Description: "forbid anonymous S3 authentication unless `-o s3.unsafe-anonymous-auth=true` is set"},
|
||||
SafeForgetKeepTags: {Type: Stable, Description: "prevent deleting all snapshots if the tag passed to `forget --keep-tags tagname` does not exist"},
|
||||
S3Restore: {Type: Alpha, Description: "restore S3 objects from cold storage classes when `-o s3.enable-restore=true` is set"},
|
||||
|
|
|
|||
Loading…
Reference in a new issue