Merge a5bcbd0568 into ccfb31b5fa

changelog/unreleased/pull-5678

@@ -0,0 +1,10 @@
+Enhancement: add new type `tree-full` to `restic cat`
+
+`restic cat tree-full <snapshot-id[:subfolder]> generates JSON output for all
+participating tree blobs for the selected snapshot or a subfolder thereof. This
+will help when troubleshooting problems and relating directory names to JSON
+nodes in the repository,
+
+https://github.com/restic/restic/pull/5679
+https://github.com/restic/restic/pull/5678
+https://github.com/restic/restic/issues/5674

cmd/restic/cmd_cat.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"path"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -13,13 +14,15 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/progress"
+	"github.com/restic/restic/internal/walker"
 )
 
-var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"}
+var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree", "full-tree"}
 
 func newCatCommand(globalOptions *global.Options) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
+		Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder|full-tree snapshot:subfolder]",
 		Short: "Print internal objects to stdout",
 		Long: `
 The "cat" command is used to print internal objects to stdout.
@@ -73,22 +76,22 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
 		return err
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
-	if err != nil {
-		return err
-	}
-	defer unlock()
-
 	tpe := args[0]
-
 	var id restic.ID
-	if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" {
+	var err error
+	if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" && tpe != "full-tree" {
 		id, err = restic.ParseID(args[1])
 		if err != nil {
 			return errors.Fatalf("unable to parse ID: %v", err)
 		}
 	}
 
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
 	switch tpe {
 	case "config":
 		buf, err := json.MarshalIndent(repo.Config(), "", "  ")
@@ -191,7 +194,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
 
 		return errors.Fatal("blob not found")
 
-	case "tree":
+	case "tree", "full-tree":
 		sn, subfolder, err := data.FindSnapshot(ctx, repo, repo, args[1])
 		if err != nil {
 			return errors.Fatalf("could not find snapshot: %v", err)
@@ -207,14 +210,68 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
 			return err
 		}
 
-		buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil)
-		if err != nil {
+		switch tpe {
+		case "tree":
+			buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil)
+			if err != nil {
+				return err
+			}
+			_, err = term.OutputRaw().Write(buf)
 			return err
+
+		case "full-tree":
+			return buildFullTree(ctx, repo, sn, subfolder, printer)
 		}
-		_, err = term.OutputRaw().Write(buf)
-		return err
+		return nil
 
 	default:
 		return errors.Fatal("invalid type")
 	}
 }
+
+// buildFullTree will create all subdirectory entries for snapshot `sn`
+// it will walk down the tree and store it in a slice for json.MarshalIndent()
+func buildFullTree(ctx context.Context, repo restic.Repository, sn *data.Snapshot,
+	subfolder string, printer progress.Printer,
+) error {
+
+	type subdirectoryEntry struct {
+		SubTree restic.ID `json:"subtree"`
+		Path    string    `json:"path"`
+	}
+
+	type snapshotTreeInfo struct {
+		SnapshotID restic.ID           `json:"snapshot"`
+		Tree       restic.ID           `json:"root"`
+		ItemList   []subdirectoryEntry `json:"subdirecories"`
+	}
+
+	treeStruct := snapshotTreeInfo{SnapshotID: *sn.ID(), Tree: *sn.Tree}
+	err := walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
+		ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error {
+			if err != nil {
+				printer.E("Unable to load tree %s  ... which belongs to snapshot %s", parentTreeID, sn.ID())
+				return walker.ErrSkipNode
+			}
+
+			if node == nil {
+				return nil
+			} else if node.Type == "dir" {
+				treeStruct.ItemList = append(treeStruct.ItemList,
+					subdirectoryEntry{*(node.Subtree), path.Join(subfolder, nodepath)})
+			}
+			return nil
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	buf, err := json.MarshalIndent(&treeStruct, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	printer.S(string(buf))
+	return nil
+}

cmd/restic/cmd_cat_integration_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"context"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/restic/restic/internal/global"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunCat(t testing.TB, gopts global.Options, tpe string, ID string) []byte {
+	buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
+		gopts.Quiet = true
+		return runCat(ctx, gopts, []string{tpe, ID}, gopts.Term)
+	})
+	rtest.OK(t, err)
+	return buf.Bytes()
+}
+
+func TestFullTree(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+	sn := testListSnapshots(t, env.gopts, 1)[0]
+	snapID := sn.String()
+
+	// gather the tree blobs from the repository: run 'restic list blobs'
+	treeIDs := testRunListTreeBlobs(t, env.gopts)
+
+	outString := string(testRunCat(t, env.gopts, "full-tree", snapID))
+
+	rtest.Assert(t, strings.Contains(outString, `"root"`), "expected to find string 'root', but did not see it")
+	rtest.Assert(t, strings.Contains(outString, snapID), "expected to find %q, but did not see it", snapID)
+	backupPath := path.Join("0", "0", "9")
+	rtest.Assert(t, strings.Contains(outString, backupPath), "expected to find %q, but did not see it", backupPath)
+	for _, treeID := range treeIDs {
+		rtest.Assert(t, strings.Contains(outString, treeID), "expected treeID %s in output string, got nothing", treeID)
+	}
+}

cmd/restic/cmd_list_integration_test.go

@@ -22,6 +22,27 @@ func testRunList(t testing.TB, gopts global.Options, tpe string) restic.IDs {
 	return parseIDsFromReader(t, buf)
 }
 
+func testRunListTreeBlobs(t testing.TB, gopts global.Options) []string {
+	buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
+		return runList(ctx, gopts, []string{"blobs"}, gopts.Term)
+	})
+	rtest.OK(t, err)
+
+	// scan the output, collect tree IDs
+	treeIDs := []string{}
+	sc := bufio.NewScanner(buf)
+	for sc.Scan() {
+		parts := strings.Split(sc.Text(), " ")
+		rtest.Assert(t, len(parts) == 2, "expected 2 items per line, got %d", len(parts))
+		if parts[0] != "tree" {
+			continue
+		}
+		rtest.Assert(t, len(parts[1]) == 64, "expected an ID, got %q", parts[1])
+		treeIDs = append(treeIDs, parts[1])
+	}
+	return treeIDs
+}
+
 func parseIDsFromReader(t testing.TB, rd io.Reader) restic.IDs {
 	t.Helper()
 	IDs := restic.IDs{}