restic/internal/data/tree_test.go

package data_test

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"testing"

	"github.com/restic/restic/internal/archiver"
	"github.com/restic/restic/internal/data"
	"github.com/restic/restic/internal/fs"
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"
	rtest "github.com/restic/restic/internal/test"
)

var testFiles = []struct {
	name    string
	content []byte
}{
	{"foo", []byte("bar")},
	{"bar/foo2", []byte("bar2")},
	{"bar/bla/blubb", []byte("This is just a test!\n")},
}

func createTempDir(t *testing.T) string {
	tempdir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-")
	rtest.OK(t, err)

	for _, test := range testFiles {
		file := filepath.Join(tempdir, test.name)
		dir := filepath.Dir(file)
		if dir != "." {
			rtest.OK(t, os.MkdirAll(dir, 0755))
		}

		f, err := os.Create(file)
		defer func() {
			rtest.OK(t, f.Close())
		}()

		rtest.OK(t, err)

		_, err = f.Write(test.content)
		rtest.OK(t, err)
	}

	return tempdir
}

func TestTree(t *testing.T) {
	dir := createTempDir(t)
	defer func() {
		if rtest.TestCleanupTempDirs {
			rtest.RemoveAll(t, dir)
		}
	}()
}

var testNodes = []data.Node{
	{Name: "normal"},
	{Name: "with backslashes \\zzz"},
	{Name: "test utf-8 föbärß"},
	{Name: "test invalid \x00\x01\x02\x03\x04"},
	{Name: "test latin1 \x75\x6d\x6c\xe4\xfc\x74\xf6\x6e\xdf\x6e\x6c\x6c"},
}

func TestNodeMarshal(t *testing.T) {
	for i, n := range testNodes {
		nodeData, err := json.Marshal(&n)
		rtest.OK(t, err)

		var node data.Node
		err = json.Unmarshal(nodeData, &node)
		rtest.OK(t, err)

		if n.Name != node.Name {
			t.Fatalf("Node %d: Names are not equal, want: %q got: %q", i, n.Name, node.Name)
		}
	}
}

func nodeForFile(t *testing.T, name string) *data.Node {
	f, err := (&fs.Local{}).OpenFile(name, fs.O_NOFOLLOW, true)
	rtest.OK(t, err)
	node, err := f.ToNode(false, t.Logf)
	rtest.OK(t, err)
	rtest.OK(t, f.Close())
	return node
}

func TestNodeComparison(t *testing.T) {
	node := nodeForFile(t, "tree_test.go")

	n2 := *node
	rtest.Assert(t, node.Equals(n2), "nodes aren't equal")

	n2.Size--
	rtest.Assert(t, !node.Equals(n2), "nodes are equal")
}

func TestEmptyLoadTree(t *testing.T) {
	repo := repository.TestRepository(t)

	tree := data.NewTree(0)
	var id restic.ID
	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
		var err error
		// save tree
		id, err = data.SaveTree(ctx, uploader, tree)
		return err
	}))

	// load tree again
	tree2, err := data.LoadTree(context.TODO(), repo, id)
	rtest.OK(t, err)

	rtest.Assert(t, tree.Equals(tree2),
		"trees are not equal: want %v, got %v",
		tree, tree2)
}

func TestTreeEqualSerialization(t *testing.T) {
	files := []string{"node.go", "tree.go", "tree_test.go"}
	for i := 1; i <= len(files); i++ {
		tree := data.NewTree(i)
		builder := data.NewTreeJSONBuilder()

		for _, fn := range files[:i] {
			node := nodeForFile(t, fn)

			rtest.OK(t, tree.Insert(node))
			rtest.OK(t, builder.AddNode(node))

			rtest.Assert(t, tree.Insert(node) != nil, "no error on duplicate node")
			rtest.Assert(t, builder.AddNode(node) != nil, "no error on duplicate node")
			rtest.Assert(t, errors.Is(builder.AddNode(node), data.ErrTreeNotOrdered), "wrong error returned")
		}

		treeBytes, err := json.Marshal(tree)
		treeBytes = append(treeBytes, '\n')
		rtest.OK(t, err)

		stiBytes, err := builder.Finalize()
		rtest.OK(t, err)

		// compare serialization of an individual node and the SaveTreeIterator
		rtest.Equals(t, treeBytes, stiBytes)
	}
}

func BenchmarkBuildTree(b *testing.B) {
	const size = 100 // Directories of this size are not uncommon.

	nodes := make([]data.Node, size)
	for i := range nodes {
		// Archiver.SaveTree inputs in sorted order, so do that here too.
		nodes[i].Name = strconv.Itoa(i)
	}

	b.ResetTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		t := data.NewTree(size)

		for i := range nodes {
			_ = t.Insert(&nodes[i])
		}
	}
}

func TestLoadTree(t *testing.T) {
	repository.TestAllVersions(t, testLoadTree)
}

func testLoadTree(t *testing.T, version uint) {
	if rtest.BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping")
	}

	// archive a few files
	repo, _, _ := repository.TestRepositoryWithVersion(t, version)
	sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil)

	_, err := data.LoadTree(context.TODO(), repo, *sn.Tree)
	rtest.OK(t, err)
}

func BenchmarkLoadTree(t *testing.B) {
	repository.BenchmarkAllVersions(t, benchmarkLoadTree)
}

func benchmarkLoadTree(t *testing.B, version uint) {
	if rtest.BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping")
	}

	// archive a few files
	repo, _, _ := repository.TestRepositoryWithVersion(t, version)
	sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil)

	t.ResetTimer()

	for i := 0; i < t.N; i++ {
		_, err := data.LoadTree(context.TODO(), repo, *sn.Tree)
		rtest.OK(t, err)
	}
}

func TestFindTreeDirectory(t *testing.T) {
	repo := repository.TestRepository(t)
	sn := data.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3)

	for _, exp := range []struct {
		subfolder string
		id        restic.ID
		err       error
	}{
		{"", restic.TestParseID("8804a5505fc3012e7d08b2843e9bda1bf3dc7644f64b542470340e1b4059f09f"), nil},
		{"/", restic.TestParseID("8804a5505fc3012e7d08b2843e9bda1bf3dc7644f64b542470340e1b4059f09f"), nil},
		{".", restic.TestParseID("8804a5505fc3012e7d08b2843e9bda1bf3dc7644f64b542470340e1b4059f09f"), nil},
		{"..", restic.ID{}, errors.New("path ..: not found")},
		{"file-1", restic.ID{}, errors.New("path file-1: not a directory")},
		{"dir-7", restic.TestParseID("1af51eb70cd4457d51db40d649bb75446a3eaa29b265916d411bb7ae971d4849"), nil},
		{"/dir-7", restic.TestParseID("1af51eb70cd4457d51db40d649bb75446a3eaa29b265916d411bb7ae971d4849"), nil},
		{"dir-7/", restic.TestParseID("1af51eb70cd4457d51db40d649bb75446a3eaa29b265916d411bb7ae971d4849"), nil},
		{"dir-7/dir-5", restic.TestParseID("f05534d2673964de698860e5069da1ee3c198acf21c187975c6feb49feb8e9c9"), nil},
	} {
		t.Run("", func(t *testing.T) {
			id, err := data.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder)
			if exp.err == nil {
				rtest.OK(t, err)
				rtest.Assert(t, exp.id == *id, "unexpected id, expected %v, got %v", exp.id, id)
			} else {
				rtest.Assert(t, exp.err.Error() == err.Error(), "unexpected err, expected %v, got %v", exp.err, err)
			}
		})
	}

	_, err := data.FindTreeDirectory(context.TODO(), repo, nil, "")
	rtest.Assert(t, err != nil, "missing error on null tree id")
}