data: use data.TreeWriter to serialize&write data.Tree

Always serialize trees via TreeJSONBuilder. Add a wrapper called
TreeWriter which combines serialization and saving the tree blob in the
repository. In the future, TreeJSONBuilder will have to upload tree
chunks while the tree is still serialized. This will a wrapper like
TreeWriter, so add it right now already.

The archiver.treeSaver still directly uses the TreeJSONBuilder as it
requires special handling.

cmd/restic/cmd_recover.go

@@ -134,28 +134,28 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
 		return ctx.Err()
 	}
 
-	tree := data.NewTree(len(roots))
-	for id := range roots {
-		var subtreeID = id
-		node := data.Node{
-			Type:       data.NodeTypeDir,
-			Name:       id.Str(),
-			Mode:       0755,
-			Subtree:    &subtreeID,
-			AccessTime: time.Now(),
-			ModTime:    time.Now(),
-			ChangeTime: time.Now(),
-		}
-		err := tree.Insert(&node)
-		if err != nil {
-			return err
-		}
-	}
-
 	var treeID restic.ID
 	err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
 		var err error
-		treeID, err = data.SaveTree(ctx, uploader, tree)
+		tw := data.NewTreeWriter(uploader)
+		for id := range roots {
+			var subtreeID = id
+			node := data.Node{
+				Type:       data.NodeTypeDir,
+				Name:       id.Str(),
+				Mode:       0755,
+				Subtree:    &subtreeID,
+				AccessTime: time.Now(),
+				ModTime:    time.Now(),
+				ChangeTime: time.Now(),
+			}
+			err := tw.AddNode(&node)
+			if err != nil {
+				return err
+			}
+		}
+
+		treeID, err = tw.Finalize(ctx)
 		if err != nil {
 			return errors.Fatalf("unable to save new tree to the repository: %v", err)
 		}

internal/data/tree.go

@@ -15,6 +15,8 @@ import (
 	"github.com/restic/restic/internal/debug"
 )
 
+var ErrTreeNotOrdered = errors.New("nodes are not ordered or duplicate")
+
 // Tree is an ordered list of nodes.
 type Tree struct {
 	Nodes []*Node `json:"nodes"`
@@ -123,28 +125,39 @@ func LoadTree(ctx context.Context, r restic.BlobLoader, id restic.ID) (*Tree, er
 	return t, nil
 }
 
-// SaveTree stores a tree into the repository and returns the ID. The ID is
-// checked against the index. The tree is only stored when the index does not
-// contain the ID.
-func SaveTree(ctx context.Context, r restic.BlobSaver, t *Tree) (restic.ID, error) {
-	if t.Nodes == nil {
-		// serialize an empty tree as `{"nodes":[]}` to be consistent with TreeJSONBuilder
-		t.Nodes = make([]*Node, 0)
-	}
-	buf, err := json.Marshal(t)
+type TreeWriter struct {
+	builder *TreeJSONBuilder
+	saver   restic.BlobSaver
+}
+
+func NewTreeWriter(saver restic.BlobSaver) *TreeWriter {
+	builder := NewTreeJSONBuilder()
+	return &TreeWriter{builder: builder, saver: saver}
+}
+
+func (t *TreeWriter) AddNode(node *Node) error {
+	return t.builder.AddNode(node)
+}
+
+func (t *TreeWriter) Finalize(ctx context.Context) (restic.ID, error) {
+	buf, err := t.builder.Finalize()
 	if err != nil {
-		return restic.ID{}, errors.Wrap(err, "MarshalJSON")
+		return restic.ID{}, err
 	}
-
-	// append a newline so that the data is always consistent (json.Encoder
-	// adds a newline after each object)
-	buf = append(buf, '\n')
-
-	id, _, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false)
+	id, _, _, err := t.saver.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false)
 	return id, err
 }
 
-var ErrTreeNotOrdered = errors.New("nodes are not ordered or duplicate")
+func SaveTree(ctx context.Context, saver restic.BlobSaver, t *Tree) (restic.ID, error) {
+	treeWriter := NewTreeWriter(saver)
+	for _, node := range t.Nodes {
+		err := treeWriter.AddNode(node)
+		if err != nil {
+			return restic.ID{}, err
+		}
+	}
+	return treeWriter.Finalize(ctx)
+}
 
 type TreeJSONBuilder struct {
 	buf      bytes.Buffer

internal/walker/rewriter.go

@@ -121,7 +121,7 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, loader restic.BlobLoader
 
 	debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str())
 
-	tb := data.NewTreeJSONBuilder()
+	tb := data.NewTreeWriter(saver)
 	for _, node := range curTree.Nodes {
 		if ctx.Err() != nil {
 			return restic.ID{}, ctx.Err()
@@ -156,13 +156,11 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, loader restic.BlobLoader
 		}
 	}
 
-	tree, err := tb.Finalize()
+	newTreeID, err := tb.Finalize(ctx)
 	if err != nil {
 		return restic.ID{}, err
 	}
 
-	// Save new tree
-	newTreeID, _, _, err := saver.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false)
 	if t.replaces != nil {
 		t.replaces[nodeID] = newTreeID
 	}