diff --git a/changelog/unreleased/pull-5678 b/changelog/unreleased/pull-5678 new file mode 100644 index 000000000..f799b9866 --- /dev/null +++ b/changelog/unreleased/pull-5678 @@ -0,0 +1,10 @@ +Enhancement: add new type `tree-full` to `restic cat` + +`restic cat tree-full generates JSON output for all +participating tree blobs for the selected snapshot or a subfolder thereof. This +will help when troubleshooting problems and relating directory names to JSON +nodes in the repository, + +https://github.com/restic/restic/pull/5679 +https://github.com/restic/restic/pull/5678 +https://github.com/restic/restic/issues/5674 diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 0d64dbd87..753c75ee5 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "path" "strings" "github.com/spf13/cobra" @@ -13,13 +14,15 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/progress" + "github.com/restic/restic/internal/walker" ) -var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"} +var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree", "full-tree"} func newCatCommand(globalOptions *global.Options) *cobra.Command { cmd := &cobra.Command{ - Use: "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]", + Use: "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder|full-tree snapshot:subfolder]", Short: "Print internal objects to stdout", Long: ` The "cat" command is used to print internal objects to stdout. @@ -73,22 +76,22 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te return err } - ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer) - if err != nil { - return err - } - defer unlock() - tpe := args[0] - var id restic.ID - if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" { + var err error + if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" && tpe != "full-tree" { id, err = restic.ParseID(args[1]) if err != nil { return errors.Fatalf("unable to parse ID: %v", err) } } + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer) + if err != nil { + return err + } + defer unlock() + switch tpe { case "config": buf, err := json.MarshalIndent(repo.Config(), "", " ") @@ -191,7 +194,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te return errors.Fatal("blob not found") - case "tree": + case "tree", "full-tree": sn, subfolder, err := data.FindSnapshot(ctx, repo, repo, args[1]) if err != nil { return errors.Fatalf("could not find snapshot: %v", err) @@ -207,14 +210,68 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te return err } - buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil) - if err != nil { + switch tpe { + case "tree": + buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil) + if err != nil { + return err + } + _, err = term.OutputRaw().Write(buf) return err + + case "full-tree": + return buildFullTree(ctx, repo, sn, subfolder, printer) } - _, err = term.OutputRaw().Write(buf) - return err + return nil default: return errors.Fatal("invalid type") } } + +// buildFullTree will create all subdirectory entries for snapshot `sn` +// it will walk down the tree and store it in a slice for json.MarshalIndent() +func buildFullTree(ctx context.Context, repo restic.Repository, sn *data.Snapshot, + subfolder string, printer progress.Printer, +) error { + + type subdirectoryEntry struct { + SubTree restic.ID `json:"subtree"` + Path string `json:"path"` + } + + type snapshotTreeInfo struct { + SnapshotID restic.ID `json:"snapshot"` + Tree restic.ID `json:"root"` + ItemList []subdirectoryEntry `json:"subdirecories"` + } + + treeStruct := snapshotTreeInfo{SnapshotID: *sn.ID(), Tree: *sn.Tree} + err := walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{ + ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error { + if err != nil { + printer.E("Unable to load tree %s ... which belongs to snapshot %s", parentTreeID, sn.ID()) + return walker.ErrSkipNode + } + + if node == nil { + return nil + } else if node.Type == "dir" { + treeStruct.ItemList = append(treeStruct.ItemList, + subdirectoryEntry{*(node.Subtree), path.Join(subfolder, nodepath)}) + } + return nil + }, + }) + if err != nil { + return err + } + + buf, err := json.MarshalIndent(&treeStruct, "", " ") + if err != nil { + return err + } + + printer.S(string(buf)) + return nil +} diff --git a/cmd/restic/cmd_cat_integration_test.go b/cmd/restic/cmd_cat_integration_test.go new file mode 100644 index 000000000..4568f4ac1 --- /dev/null +++ b/cmd/restic/cmd_cat_integration_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "context" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/restic/restic/internal/global" + rtest "github.com/restic/restic/internal/test" +) + +func testRunCat(t testing.TB, gopts global.Options, tpe string, ID string) []byte { + buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error { + gopts.Quiet = true + return runCat(ctx, gopts, []string{tpe, ID}, gopts.Term) + }) + rtest.OK(t, err) + return buf.Bytes() +} + +func TestFullTree(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{} + + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts) + testRunCheck(t, env.gopts) + sn := testListSnapshots(t, env.gopts, 1)[0] + snapID := sn.String() + + // gather the tree blobs from the repository: run 'restic list blobs' + treeIDs := testRunListTreeBlobs(t, env.gopts) + + outString := string(testRunCat(t, env.gopts, "full-tree", snapID)) + + rtest.Assert(t, strings.Contains(outString, `"root"`), "expected to find string 'root', but did not see it") + rtest.Assert(t, strings.Contains(outString, snapID), "expected to find %q, but did not see it", snapID) + backupPath := path.Join("0", "0", "9") + rtest.Assert(t, strings.Contains(outString, backupPath), "expected to find %q, but did not see it", backupPath) + for _, treeID := range treeIDs { + rtest.Assert(t, strings.Contains(outString, treeID), "expected treeID %s in output string, got nothing", treeID) + } +} diff --git a/cmd/restic/cmd_list_integration_test.go b/cmd/restic/cmd_list_integration_test.go index 655d6da27..3ec695c58 100644 --- a/cmd/restic/cmd_list_integration_test.go +++ b/cmd/restic/cmd_list_integration_test.go @@ -22,6 +22,27 @@ func testRunList(t testing.TB, gopts global.Options, tpe string) restic.IDs { return parseIDsFromReader(t, buf) } +func testRunListTreeBlobs(t testing.TB, gopts global.Options) []string { + buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error { + return runList(ctx, gopts, []string{"blobs"}, gopts.Term) + }) + rtest.OK(t, err) + + // scan the output, collect tree IDs + treeIDs := []string{} + sc := bufio.NewScanner(buf) + for sc.Scan() { + parts := strings.Split(sc.Text(), " ") + rtest.Assert(t, len(parts) == 2, "expected 2 items per line, got %d", len(parts)) + if parts[0] != "tree" { + continue + } + rtest.Assert(t, len(parts[1]) == 64, "expected an ID, got %q", parts[1]) + treeIDs = append(treeIDs, parts[1]) + } + return treeIDs +} + func parseIDsFromReader(t testing.TB, rd io.Reader) restic.IDs { t.Helper() IDs := restic.IDs{}