mirror of
https://github.com/restic/restic.git
synced 2026-02-03 12:29:37 -05:00
5148608c39
cmd/restic/cmd_rewrite.go: introduction of include filters for this command: - add include filters, add error checking code - add new parameter 'keepEmptyDirectoryFunc' to 'walker.NewSnapshotSizeRewriter()', so empty directories have to be kept to keep the directory structure intact - add parameter 'keepEmptySnapshot' to 'filterAndReplaceSnapshot()' to keep snapshots intact when nothing is to be included - introduce helper function 'gatherIncludeFilters()' and 'gatherExcludeFilters()' to keep code flow clean cmd/restic/cmd_rewrite_integration_test.go: add several new tests around the 'include' functionality internal/filter/include.go: this is where is include filter is defined internal/walker/rewriter.go: - struct RewriteOpts gains field 'KeepEmtpyDirectory', which is a 'NodeKeepEmptyDirectoryFunc()' which defaults to nil, so that al subdirectories are kept - function 'NewSnapshotSizeRewriter()' gains the parameter 'keepEmptyDirecoryFilter' which controls the management of empty subdirectories in case of include filters active internal/data/tree.go: gains a function Count() for checking the number if node elements in a newly built tree internal/walker/rewriter_test.go: function 'NewSnapshotSizeRewriter()' gets an additional parameter nil to keeps things happy cmd/restic/cmd_repair_snapshots.go: function 'filterAndReplaceSnapshot()' gets an additional parameter 'keepEmptySnapshot=nil' doc/045_working_with_repos.rst: gets to mention include filters changelog/unreleased/issue-4278: the usual announcement file git rebase master -i produced this restic rewrite include - keep linter happy cmd/restic/cmd_rewrite_integration_test.go: linter likes strings.Contain() better than my strings.Index() >= 0
394 lines
9.2 KiB
Go
394 lines
9.2 KiB
Go
package walker
|
|
|
|
import (
|
|
"context"
|
|
"slices"
|
|
"testing"
|
|
|
|
"github.com/restic/restic/internal/data"
|
|
"github.com/restic/restic/internal/restic"
|
|
"github.com/restic/restic/internal/test"
|
|
)
|
|
|
|
type checkRewriteFunc func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB))
|
|
|
|
// checkRewriteItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'.
|
|
func checkRewriteItemOrder(want []string) checkRewriteFunc {
|
|
pos := 0
|
|
return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) {
|
|
rewriter = NewTreeRewriter(RewriteOpts{
|
|
RewriteNode: func(node *data.Node, path string) *data.Node {
|
|
if pos >= len(want) {
|
|
t.Errorf("additional unexpected path found: %v", path)
|
|
return nil
|
|
}
|
|
|
|
if path != want[pos] {
|
|
t.Errorf("wrong path found, want %q, got %q", want[pos], path)
|
|
}
|
|
pos++
|
|
return node
|
|
},
|
|
})
|
|
|
|
final = func(t testing.TB) {
|
|
if pos != len(want) {
|
|
t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
|
|
}
|
|
}
|
|
|
|
return rewriter, final
|
|
}
|
|
}
|
|
|
|
// checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceeds in the correct order.
|
|
func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache bool) checkRewriteFunc {
|
|
var pos int
|
|
|
|
return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) {
|
|
rewriter = NewTreeRewriter(RewriteOpts{
|
|
RewriteNode: func(node *data.Node, path string) *data.Node {
|
|
if pos >= len(want) {
|
|
t.Errorf("additional unexpected path found: %v", path)
|
|
return nil
|
|
}
|
|
|
|
if path != want[pos] {
|
|
t.Errorf("wrong path found, want %q, got %q", want[pos], path)
|
|
}
|
|
pos++
|
|
|
|
_, skip := skipFor[path]
|
|
if skip {
|
|
return nil
|
|
}
|
|
return node
|
|
},
|
|
DisableNodeCache: disableCache,
|
|
})
|
|
|
|
final = func(t testing.TB) {
|
|
if pos != len(want) {
|
|
t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
|
|
}
|
|
}
|
|
|
|
return rewriter, final
|
|
}
|
|
}
|
|
|
|
// checkIncreaseNodeSize modifies each node by changing its size.
|
|
func checkIncreaseNodeSize(increase uint64) checkRewriteFunc {
|
|
return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) {
|
|
rewriter = NewTreeRewriter(RewriteOpts{
|
|
RewriteNode: func(node *data.Node, path string) *data.Node {
|
|
if node.Type == data.NodeTypeFile {
|
|
node.Size += increase
|
|
}
|
|
return node
|
|
},
|
|
})
|
|
|
|
final = func(t testing.TB) {}
|
|
|
|
return rewriter, final
|
|
}
|
|
}
|
|
|
|
func TestRewriter(t *testing.T) {
|
|
var tests = []struct {
|
|
tree TestTree
|
|
newTree TestTree
|
|
check checkRewriteFunc
|
|
}{
|
|
{ // don't change
|
|
tree: TestTree{
|
|
"foo": TestFile{},
|
|
"subdir": TestTree{
|
|
"subfile": TestFile{},
|
|
},
|
|
},
|
|
check: checkRewriteItemOrder([]string{
|
|
"/foo",
|
|
"/subdir",
|
|
"/subdir/subfile",
|
|
}),
|
|
},
|
|
{ // exclude file
|
|
tree: TestTree{
|
|
"foo": TestFile{},
|
|
"subdir": TestTree{
|
|
"subfile": TestFile{},
|
|
},
|
|
},
|
|
newTree: TestTree{
|
|
"foo": TestFile{},
|
|
"subdir": TestTree{},
|
|
},
|
|
check: checkRewriteSkips(
|
|
map[string]struct{}{
|
|
"/subdir/subfile": {},
|
|
},
|
|
[]string{
|
|
"/foo",
|
|
"/subdir",
|
|
"/subdir/subfile",
|
|
},
|
|
false,
|
|
),
|
|
},
|
|
{ // exclude dir
|
|
tree: TestTree{
|
|
"foo": TestFile{},
|
|
"subdir": TestTree{
|
|
"subfile": TestFile{},
|
|
},
|
|
},
|
|
newTree: TestTree{
|
|
"foo": TestFile{},
|
|
},
|
|
check: checkRewriteSkips(
|
|
map[string]struct{}{
|
|
"/subdir": {},
|
|
},
|
|
[]string{
|
|
"/foo",
|
|
"/subdir",
|
|
},
|
|
false,
|
|
),
|
|
},
|
|
{ // modify node
|
|
tree: TestTree{
|
|
"foo": TestFile{Size: 21},
|
|
"subdir": TestTree{
|
|
"subfile": TestFile{Size: 21},
|
|
},
|
|
},
|
|
newTree: TestTree{
|
|
"foo": TestFile{Size: 42},
|
|
"subdir": TestTree{
|
|
"subfile": TestFile{Size: 42},
|
|
},
|
|
},
|
|
check: checkIncreaseNodeSize(21),
|
|
},
|
|
{ // test cache
|
|
tree: TestTree{
|
|
// both subdirs are identical
|
|
"subdir1": TestTree{
|
|
"subfile": TestFile{},
|
|
"subfile2": TestFile{},
|
|
},
|
|
"subdir2": TestTree{
|
|
"subfile": TestFile{},
|
|
"subfile2": TestFile{},
|
|
},
|
|
},
|
|
newTree: TestTree{
|
|
"subdir1": TestTree{
|
|
"subfile2": TestFile{},
|
|
},
|
|
"subdir2": TestTree{
|
|
"subfile2": TestFile{},
|
|
},
|
|
},
|
|
check: checkRewriteSkips(
|
|
map[string]struct{}{
|
|
"/subdir1/subfile": {},
|
|
},
|
|
[]string{
|
|
"/subdir1",
|
|
"/subdir1/subfile",
|
|
"/subdir1/subfile2",
|
|
"/subdir2",
|
|
},
|
|
false,
|
|
),
|
|
},
|
|
{ // test disabled cache
|
|
tree: TestTree{
|
|
// both subdirs are identical
|
|
"subdir1": TestTree{
|
|
"subfile": TestFile{},
|
|
"subfile2": TestFile{},
|
|
},
|
|
"subdir2": TestTree{
|
|
"subfile": TestFile{},
|
|
"subfile2": TestFile{},
|
|
},
|
|
},
|
|
newTree: TestTree{
|
|
"subdir1": TestTree{
|
|
"subfile2": TestFile{},
|
|
},
|
|
"subdir2": TestTree{
|
|
"subfile": TestFile{},
|
|
"subfile2": TestFile{},
|
|
},
|
|
},
|
|
check: checkRewriteSkips(
|
|
map[string]struct{}{
|
|
"/subdir1/subfile": {},
|
|
},
|
|
[]string{
|
|
"/subdir1",
|
|
"/subdir1/subfile",
|
|
"/subdir1/subfile2",
|
|
"/subdir2",
|
|
"/subdir2/subfile",
|
|
"/subdir2/subfile2",
|
|
},
|
|
true,
|
|
),
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
repo, root := BuildTreeMap(test.tree)
|
|
if test.newTree == nil {
|
|
test.newTree = test.tree
|
|
}
|
|
expRepo, expRoot := BuildTreeMap(test.newTree)
|
|
modrepo := data.TestWritableTreeMap{TestTreeMap: repo}
|
|
|
|
ctx, cancel := context.WithCancel(context.TODO())
|
|
defer cancel()
|
|
|
|
rewriter, last := test.check(t)
|
|
newRoot, err := rewriter.RewriteTree(ctx, modrepo, modrepo, "/", root)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
last(t)
|
|
|
|
// verifying against the expected tree root also implicitly checks the structural integrity
|
|
if newRoot != expRoot {
|
|
t.Error("hash mismatch")
|
|
t.Log("Got")
|
|
modrepo.Dump(t)
|
|
t.Log("Expected")
|
|
data.TestWritableTreeMap{TestTreeMap: expRepo}.Dump(t)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSnapshotSizeQuery(t *testing.T) {
|
|
tree := TestTree{
|
|
"foo": TestFile{Size: 21},
|
|
"bar": TestFile{Size: 21},
|
|
"subdir": TestTree{
|
|
"subfile": TestFile{Size: 21},
|
|
},
|
|
}
|
|
newTree := TestTree{
|
|
"foo": TestFile{Size: 42},
|
|
"subdir": TestTree{
|
|
"subfile": TestFile{Size: 42},
|
|
},
|
|
}
|
|
t.Run("", func(t *testing.T) {
|
|
repo, root := BuildTreeMap(tree)
|
|
expRepo, expRoot := BuildTreeMap(newTree)
|
|
modrepo := data.TestWritableTreeMap{TestTreeMap: repo}
|
|
|
|
ctx, cancel := context.WithCancel(context.TODO())
|
|
defer cancel()
|
|
|
|
rewriteNode := func(node *data.Node, path string) *data.Node {
|
|
if path == "/bar" {
|
|
return nil
|
|
}
|
|
if node.Type == data.NodeTypeFile {
|
|
node.Size += 21
|
|
}
|
|
return node
|
|
}
|
|
rewriter, querySize := NewSnapshotSizeRewriter(rewriteNode, nil)
|
|
newRoot, err := rewriter.RewriteTree(ctx, modrepo, modrepo, "/", root)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
|
|
ss := querySize()
|
|
|
|
test.Equals(t, uint(2), ss.FileCount, "snapshot file count mismatch")
|
|
test.Equals(t, uint64(84), ss.FileSize, "snapshot size mismatch")
|
|
|
|
// verifying against the expected tree root also implicitly checks the structural integrity
|
|
if newRoot != expRoot {
|
|
t.Error("hash mismatch")
|
|
t.Log("Got")
|
|
modrepo.Dump(t)
|
|
t.Log("Expected")
|
|
data.TestWritableTreeMap{TestTreeMap: expRepo}.Dump(t)
|
|
}
|
|
})
|
|
|
|
}
|
|
|
|
func TestRewriterFailOnUnknownFields(t *testing.T) {
|
|
tm := data.TestWritableTreeMap{TestTreeMap: data.TestTreeMap{}}
|
|
node := []byte(`{"nodes":[{"name":"subfile","type":"file","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","uid":0,"gid":0,"content":null,"unknown_field":42}]}`)
|
|
id := restic.Hash(node)
|
|
tm.TestTreeMap[id] = node
|
|
|
|
ctx, cancel := context.WithCancel(context.TODO())
|
|
defer cancel()
|
|
|
|
rewriter := NewTreeRewriter(RewriteOpts{
|
|
RewriteNode: func(node *data.Node, path string) *data.Node {
|
|
// tree loading must not succeed
|
|
t.Fail()
|
|
return node
|
|
},
|
|
})
|
|
_, err := rewriter.RewriteTree(ctx, tm, tm, "/", id)
|
|
|
|
if err == nil {
|
|
t.Error("missing error on unknown field")
|
|
}
|
|
|
|
// check that the serialization check can be disabled
|
|
rewriter = NewTreeRewriter(RewriteOpts{
|
|
AllowUnstableSerialization: true,
|
|
})
|
|
root, err := rewriter.RewriteTree(ctx, tm, tm, "/", id)
|
|
test.OK(t, err)
|
|
_, expRoot := BuildTreeMap(TestTree{
|
|
"subfile": TestFile{},
|
|
})
|
|
test.Assert(t, root == expRoot, "mismatched trees")
|
|
}
|
|
|
|
func TestRewriterTreeLoadError(t *testing.T) {
|
|
tm := data.TestWritableTreeMap{TestTreeMap: data.TestTreeMap{}}
|
|
id := restic.NewRandomID()
|
|
|
|
ctx, cancel := context.WithCancel(context.TODO())
|
|
defer cancel()
|
|
|
|
// also check that load error by default cause the operation to fail
|
|
rewriter := NewTreeRewriter(RewriteOpts{})
|
|
_, err := rewriter.RewriteTree(ctx, tm, tm, "/", id)
|
|
if err == nil {
|
|
t.Fatal("missing error on unloadable tree")
|
|
}
|
|
|
|
replacementNode := &data.Node{Name: "replacement", Type: data.NodeTypeFile, Size: 42}
|
|
replacementID := data.TestSaveNodes(t, ctx, tm, []*data.Node{replacementNode})
|
|
|
|
rewriter = NewTreeRewriter(RewriteOpts{
|
|
RewriteFailedTree: func(nodeID restic.ID, path string, err error) (data.TreeNodeIterator, error) {
|
|
if nodeID != id || path != "/" {
|
|
t.Fail()
|
|
}
|
|
return slices.Values([]data.NodeOrError{{Node: replacementNode}}), nil
|
|
},
|
|
})
|
|
newRoot, err := rewriter.RewriteTree(ctx, tm, tm, "/", id)
|
|
test.OK(t, err)
|
|
test.Equals(t, replacementID, newRoot)
|
|
}
|