dump: add progress reporter when --target is set

This commit is contained in:
Srigovind Nayak 2026-05-16 22:35:57 +05:30
parent 990329013e
commit 383075bba1
No known key found for this signature in database
GPG key ID: A5A62F4DD79FB9E5
12 changed files with 631 additions and 6 deletions

View file

@ -0,0 +1,8 @@
Enhancement: Add progress reporting for `dump` command
Restic now reports progress for the `dump` command when the `--target` flag
is specified. Progress reporting is also available with the `--json` flag.
At the end of the `dump` command, a summary of the progress is reported.
https://github.com/restic/restic/issues/5309
https://github.com/restic/restic/pull/5366

View file

@ -14,6 +14,7 @@ import (
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
dumpui "github.com/restic/restic/internal/ui/dump"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -180,6 +181,8 @@ func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args [
outputFileWriter := term.OutputRaw()
canWriteArchiveFunc := checkStdoutArchive(term)
// Setup progress reporting for target file
var progressReporter *dumpui.Progress
if opts.Target != "" {
file, err := os.Create(opts.Target)
if err != nil {
@ -191,9 +194,22 @@ func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args [
outputFileWriter = file
canWriteArchiveFunc = func() error { return nil }
// Initialize progress reporting only when writing to a file
var progressPrinter dumpui.ProgressPrinter
if gopts.JSON {
progressPrinter = dumpui.NewJSONProgress(term, gopts.Verbosity)
} else {
progressPrinter = dumpui.NewTextProgress(term, gopts.Verbosity)
}
progressReporter = dumpui.NewProgress(progressPrinter, ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()))
defer progressReporter.Finish()
}
d := dump.New(opts.Archive, repo, outputFileWriter)
if progressReporter != nil {
d.SetProgressReporter(progressReporter)
}
err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc)
if err != nil {
return errors.Fatalf("cannot dump file: %v", err)

View file

@ -169,7 +169,7 @@ command:
-v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2)
Subcommands that support showing progress information such as ``backup``,
``restore``, ``check`` and ``prune`` will do so unless the quiet flag ``-q``
``restore``, ``check``, ``dump`` and ``prune`` will do so unless the quiet flag ``-q``
or ``--quiet`` is set. When running from a non-interactive console progress
reporting is disabled by default to not fill your logs. For interactive and
non-interactive consoles the environment variable ``RESTIC_PROGRESS_FPS`` can

View file

@ -15,10 +15,18 @@ import (
// A Dumper writes trees and files from a repository to a Writer
// in an archive format.
type Dumper struct {
cache *bloblru.Cache
format string
repo restic.Loader
w io.Writer
cache *bloblru.Cache
format string
repo restic.Loader
w io.Writer
progress ProgressReporter
}
// ProgressReporter is an interface for reporting progress during dump operations
type ProgressReporter interface {
// AddProgress reports progress for any node (file, directory, symlink)
AddProgress(item string, size uint64, nodeType data.NodeType)
Error(item string, err error) error
}
func New(format string, repo restic.Loader, w io.Writer) *Dumper {
@ -30,6 +38,11 @@ func New(format string, repo restic.Loader, w io.Writer) *Dumper {
}
}
// SetProgressReporter sets a progress reporter for the dumper
func (d *Dumper) SetProgressReporter(progress ProgressReporter) {
d.progress = progress
}
func (d *Dumper) DumpTree(ctx context.Context, tree data.TreeNodeIterator, rootPath string) error {
wg, ctx := errgroup.WithContext(ctx)
@ -109,7 +122,12 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *data.Node, ch
// WriteNode writes a file node's contents directly to d's Writer,
// without caring about d's format.
func (d *Dumper) WriteNode(ctx context.Context, node *data.Node) error {
return d.writeNode(ctx, d.w, node)
err := d.writeNode(ctx, d.w, node)
if err == nil && d.progress != nil {
// Report progress for all node types
d.progress.AddProgress(node.Path, node.Size, node.Type)
}
return err
}
func (d *Dumper) writeNode(ctx context.Context, w io.Writer, node *data.Node) error {
@ -126,6 +144,9 @@ func (d *Dumper) writeNode(ctx context.Context, w io.Writer, node *data.Node) er
return ctx.Err()
case blob := <-ch:
if _, err := w.Write(blob); err != nil {
if d.progress != nil {
_ = d.progress.Error(node.Path, err)
}
return err
}
}

View file

@ -26,6 +26,9 @@ func (d *Dumper) dumpTar(ctx context.Context, ch <-chan *data.Node) (err error)
if err := d.dumpNodeTar(ctx, node, w); err != nil {
return err
}
if d.progress != nil {
d.progress.AddProgress(node.Path, node.Size, node.Type)
}
}
return nil
}
@ -51,6 +54,9 @@ func tarIdentifier(id uint32) int {
func (d *Dumper) dumpNodeTar(ctx context.Context, node *data.Node, w *tar.Writer) error {
relPath, err := filepath.Rel("/", node.Path)
if err != nil {
if d.progress != nil {
_ = d.progress.Error(node.Path, err)
}
return err
}
@ -95,6 +101,9 @@ func (d *Dumper) dumpNodeTar(ctx context.Context, node *data.Node, w *tar.Writer
err = w.WriteHeader(header)
if err != nil {
if d.progress != nil {
_ = d.progress.Error(node.Path, err)
}
return fmt.Errorf("writing header for %q: %w", node.Path, err)
}
return d.writeNode(ctx, w, node)

View file

@ -23,6 +23,9 @@ func (d *Dumper) dumpZip(ctx context.Context, ch <-chan *data.Node) (err error)
if err := d.dumpNodeZip(ctx, node, w); err != nil {
return err
}
if d.progress != nil {
d.progress.AddProgress(node.Path, node.Size, node.Type)
}
}
return nil
}
@ -30,6 +33,9 @@ func (d *Dumper) dumpZip(ctx context.Context, ch <-chan *data.Node) (err error)
func (d *Dumper) dumpNodeZip(ctx context.Context, node *data.Node, zw *zip.Writer) error {
relPath, err := filepath.Rel("/", node.Path)
if err != nil {
if d.progress != nil {
_ = d.progress.Error(node.Path, err)
}
return err
}
@ -49,14 +55,26 @@ func (d *Dumper) dumpNodeZip(ctx context.Context, node *data.Node, zw *zip.Write
w, err := zw.CreateHeader(header)
if err != nil {
if d.progress != nil {
_ = d.progress.Error(node.Path, err)
}
return errors.Wrap(err, "ZipHeader")
}
if node.Type == data.NodeTypeSymlink {
if _, err = w.Write([]byte(node.LinkTarget)); err != nil {
if d.progress != nil {
_ = d.progress.Error(node.Path, err)
}
return errors.Wrap(err, "Write")
}
// Report progress for symlink nodes
if d.progress != nil {
// Pass the node type as well
d.progress.AddProgress(node.Path, uint64(len(node.LinkTarget)), node.Type)
}
return nil
}

118
internal/ui/dump/json.go Normal file
View file

@ -0,0 +1,118 @@
package dump
import (
"time"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
)
type jsonPrinter struct {
progress.Printer
term ui.Terminal
verbosity uint
}
// NewJSONProgress creates a new JSON-based progress printer
func NewJSONProgress(term ui.Terminal, verbosity uint) ProgressPrinter {
return &jsonPrinter{
Printer: ui.NewProgressPrinter(true, verbosity, term),
term: term,
verbosity: verbosity,
}
}
func (t *jsonPrinter) print(status interface{}) {
t.term.Print(ui.ToJSONString(status))
}
func (t *jsonPrinter) error(status interface{}) {
t.term.Error(ui.ToJSONString(status))
}
func (t *jsonPrinter) Update(state State, duration time.Duration) {
status := statusUpdate{
MessageType: "status",
SecondsElapsed: uint64(duration / time.Second),
FilesProcessed: state.FilesProcessed,
DirsProcessed: state.DirsProcessed,
TotalItems: state.TotalItems,
BytesProcessed: state.BytesProcessed,
}
t.print(status)
}
func (t *jsonPrinter) Error(item string, err error) error {
t.error(errorUpdate{
MessageType: "error",
Error: errorObject{err.Error()},
During: "dump",
Item: item,
})
return nil
}
func (t *jsonPrinter) CompleteItem(item string, size uint64, nodeType string) {
if t.verbosity < 3 {
return
}
status := verboseUpdate{
MessageType: "verbose_status",
Action: "dumped",
NodeType: nodeType,
Item: item,
Size: size,
}
t.print(status)
}
func (t *jsonPrinter) Finish(state State, duration time.Duration) {
status := summaryOutput{
MessageType: "summary",
SecondsElapsed: uint64(duration / time.Second),
FilesProcessed: state.FilesProcessed,
DirsProcessed: state.DirsProcessed,
TotalItems: state.TotalItems,
BytesProcessed: state.BytesProcessed,
}
t.print(status)
}
type statusUpdate struct {
MessageType string `json:"message_type"` // "status"
SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"`
FilesProcessed uint64 `json:"files_processed,omitempty"`
DirsProcessed uint64 `json:"dirs_processed,omitempty"`
TotalItems uint64 `json:"total_items,omitempty"`
BytesProcessed uint64 `json:"bytes_processed,omitempty"`
}
type errorObject struct {
Message string `json:"message"`
}
type errorUpdate struct {
MessageType string `json:"message_type"` // "error"
Error errorObject `json:"error"`
During string `json:"during"`
Item string `json:"item"`
}
type verboseUpdate struct {
MessageType string `json:"message_type"` // "verbose_status"
Action string `json:"action"`
NodeType string `json:"node_type"`
Item string `json:"item"`
Size uint64 `json:"size"`
}
type summaryOutput struct {
MessageType string `json:"message_type"` // "summary"
SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"`
FilesProcessed uint64 `json:"files_processed,omitempty"`
DirsProcessed uint64 `json:"dirs_processed,omitempty"`
TotalItems uint64 `json:"total_items,omitempty"`
BytesProcessed uint64 `json:"bytes_processed,omitempty"`
}

View file

@ -0,0 +1,50 @@
package dump
import (
"testing"
"time"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
)
func createJSONProgress() (*ui.MockTerminal, ProgressPrinter) {
term := &ui.MockTerminal{}
printer := NewJSONProgress(term, 3)
return term, printer
}
func TestJSONPrintUpdate(t *testing.T) {
term, printer := createJSONProgress()
printer.Update(State{10, 5, 15, 1000}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"files_processed\":10,\"dirs_processed\":5,\"total_items\":15,\"bytes_processed\":1000}\n"}, term.Output)
}
func TestJSONPrintSummary(t *testing.T) {
term, printer := createJSONProgress()
printer.Finish(State{20, 10, 30, 2000}, 10*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":10,\"files_processed\":20,\"dirs_processed\":10,\"total_items\":30,\"bytes_processed\":2000}\n"}, term.Output)
}
func TestJSONPrintCompleteItem(t *testing.T) {
for _, data := range []struct {
nodeType string
size uint64
expected string
}{
{"dir", 0, "{\"message_type\":\"verbose_status\",\"action\":\"dumped\",\"node_type\":\"dir\",\"item\":\"test\",\"size\":0}\n"},
{"file", 123, "{\"message_type\":\"verbose_status\",\"action\":\"dumped\",\"node_type\":\"file\",\"item\":\"test\",\"size\":123}\n"},
{"symlink", 0, "{\"message_type\":\"verbose_status\",\"action\":\"dumped\",\"node_type\":\"symlink\",\"item\":\"test\",\"size\":0}\n"},
} {
term, printer := createJSONProgress()
printer.CompleteItem("test", data.size, data.nodeType)
test.Equals(t, []string{data.expected}, term.Output)
}
}
func TestJSONError(t *testing.T) {
term, printer := createJSONProgress()
test.Equals(t, printer.Error("/path", errors.New("error \"message\"")), nil)
test.Equals(t, []string{"{\"message_type\":\"error\",\"error\":{\"message\":\"error \\\"message\\\"\"},\"during\":\"dump\",\"item\":\"/path\"}\n"}, term.Errors)
}

View file

@ -0,0 +1,109 @@
package dump
import (
"sync"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/ui/progress"
)
// State represents the current progress state of the dump operation
type State struct {
FilesProcessed uint64
DirsProcessed uint64
TotalItems uint64
BytesProcessed uint64
}
// ProgressPrinter is an interface for printing progress information
type ProgressPrinter interface {
Update(state State, duration time.Duration)
Error(item string, err error) error
CompleteItem(item string, size uint64, nodeType string)
Finish(state State, duration time.Duration)
progress.Printer
}
// Progress tracks and reports the progress of a dump operation
type Progress struct {
updater progress.Updater
m sync.Mutex
state State
started time.Time
printer ProgressPrinter
}
// NewProgress creates a new Progress tracker
func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
p := &Progress{
started: time.Now(),
printer: printer,
}
p.updater = *progress.NewUpdater(interval, p.update)
return p
}
func (p *Progress) update(runtime time.Duration, final bool) {
p.m.Lock()
defer p.m.Unlock()
if !final {
p.printer.Update(p.state, runtime)
} else {
p.printer.Finish(p.state, runtime)
}
}
// AddProgress records progress for a node that has been processed
func (p *Progress) AddProgress(item string, size uint64, nodeType data.NodeType) {
if p == nil {
return
}
p.m.Lock()
defer p.m.Unlock()
// Increment the appropriate counter based on node type
switch nodeType {
case data.NodeTypeFile:
p.state.FilesProcessed++
case data.NodeTypeDir:
p.state.DirsProcessed++
}
// Increment total items and bytes for all node types
p.state.TotalItems++
p.state.BytesProcessed += size
// Convert nodeType to string for the printer
nodeTypeStr := "file"
switch nodeType {
case data.NodeTypeDir:
nodeTypeStr = "dir"
case data.NodeTypeSymlink:
nodeTypeStr = "symlink"
}
p.printer.CompleteItem(item, size, nodeTypeStr)
}
// Error reports an error that occurred during the dump operation
func (p *Progress) Error(item string, err error) error {
if p == nil {
return nil
}
p.m.Lock()
defer p.m.Unlock()
return p.printer.Error(item, err)
}
// Finish completes the progress reporting
func (p *Progress) Finish() {
p.updater.Done()
}

View file

@ -0,0 +1,171 @@
package dump
import (
"testing"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/progress"
)
type printerTraceEntry struct {
progress State
duration time.Duration
isFinished bool
}
type printerTrace []printerTraceEntry
type itemTraceEntry struct {
item string
size uint64
nodeType string
}
type itemTrace []itemTraceEntry
type errorTraceEntry struct {
item string
err error
}
type errorTrace []errorTraceEntry
type mockPrinter struct {
progress.NoopPrinter
trace printerTrace
items itemTrace
errors errorTrace
}
const mockFinishDuration = 42 * time.Second
func (p *mockPrinter) Update(progress State, duration time.Duration) {
p.trace = append(p.trace, printerTraceEntry{progress, duration, false})
}
func (p *mockPrinter) Error(item string, err error) error {
p.errors = append(p.errors, errorTraceEntry{item, err})
return nil
}
func (p *mockPrinter) CompleteItem(item string, size uint64, nodeType string) {
p.items = append(p.items, itemTraceEntry{item, size, nodeType})
}
func (p *mockPrinter) Finish(progress State, _ time.Duration) {
p.trace = append(p.trace, printerTraceEntry{progress, mockFinishDuration, true})
}
func testProgress(fn func(progress *Progress) bool) (printerTrace, itemTrace, errorTrace) {
printer := &mockPrinter{}
progress := NewProgress(printer, 0)
final := fn(progress)
progress.update(0, final)
trace := append(printerTrace{}, printer.trace...)
items := append(itemTrace{}, printer.items...)
errors := append(errorTrace{}, printer.errors...)
// cleanup to avoid goroutine leak, but copy trace first
progress.Finish()
return trace, items, errors
}
func TestNew(t *testing.T) {
result, items, _ := testProgress(func(progress *Progress) bool {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{0, 0, 0, 0}, 0, false},
}, result)
test.Equals(t, itemTrace{}, items)
}
func TestAddFileProgress(t *testing.T) {
fileSize := uint64(100)
result, items, _ := testProgress(func(progress *Progress) bool {
progress.AddProgress("test.txt", fileSize, data.NodeTypeFile)
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{1, 0, 1, fileSize}, 0, false},
}, result)
test.Equals(t, itemTrace{
itemTraceEntry{item: "test.txt", size: fileSize, nodeType: "file"},
}, items)
}
func TestAddDirProgress(t *testing.T) {
result, items, _ := testProgress(func(progress *Progress) bool {
progress.AddProgress("test-dir", 0, data.NodeTypeDir)
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{0, 1, 1, 0}, 0, false},
}, result)
test.Equals(t, itemTrace{
itemTraceEntry{item: "test-dir", size: 0, nodeType: "dir"},
}, items)
}
func TestMultipleItems(t *testing.T) {
fileSize := uint64(100)
result, items, _ := testProgress(func(progress *Progress) bool {
progress.AddProgress("test-dir", 0, data.NodeTypeDir)
progress.AddProgress("test1.txt", fileSize, data.NodeTypeFile)
progress.AddProgress("test2.txt", fileSize*2, data.NodeTypeFile)
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{2, 1, 3, fileSize * 3}, 0, false},
}, result)
test.Equals(t, itemTrace{
itemTraceEntry{item: "test-dir", size: 0, nodeType: "dir"},
itemTraceEntry{item: "test1.txt", size: fileSize, nodeType: "file"},
itemTraceEntry{item: "test2.txt", size: fileSize * 2, nodeType: "file"},
}, items)
}
func TestSummaryOnFinish(t *testing.T) {
fileSize := uint64(100)
result, _, _ := testProgress(func(progress *Progress) bool {
progress.AddProgress("test-dir", 0, data.NodeTypeDir)
progress.AddProgress("test1.txt", fileSize, data.NodeTypeFile)
progress.AddProgress("test2.txt", fileSize*2, data.NodeTypeFile)
return true
})
test.Equals(t, printerTrace{
printerTraceEntry{State{2, 1, 3, fileSize * 3}, mockFinishDuration, true},
}, result)
}
func TestProgressError(t *testing.T) {
err1 := errors.New("err1")
err2 := errors.New("err2")
_, _, errors := testProgress(func(progress *Progress) bool {
test.Equals(t, progress.Error("first", err1), nil)
test.Equals(t, progress.Error("second", err2), nil)
return true
})
test.Equals(t, errorTrace{
errorTraceEntry{"first", err1},
errorTraceEntry{"second", err2},
}, errors)
}
func TestSymlinkProgress(t *testing.T) {
result, items, _ := testProgress(func(progress *Progress) bool {
progress.AddProgress("test-symlink", 0, data.NodeTypeSymlink)
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{0, 0, 1, 0}, 0, false},
}, result)
test.Equals(t, itemTrace{
itemTraceEntry{item: "test-symlink", size: 0, nodeType: "symlink"},
}, items)
}

55
internal/ui/dump/text.go Normal file
View file

@ -0,0 +1,55 @@
package dump
import (
"fmt"
"time"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
)
type textPrinter struct {
progress.Printer
term ui.Terminal
verbosity uint
}
// NewTextProgress creates a new text-based progress printer
func NewTextProgress(term ui.Terminal, verbosity uint) ProgressPrinter {
return &textPrinter{
Printer: ui.NewProgressPrinter(false, verbosity, term),
term: term,
verbosity: verbosity,
}
}
func (t *textPrinter) Update(state State, duration time.Duration) {
timeElapsed := ui.FormatDuration(duration)
formattedBytesProcessed := ui.FormatBytes(state.BytesProcessed)
progress := fmt.Sprintf("[%s] %v files, %v dirs, %v total items %s",
timeElapsed, state.FilesProcessed, state.DirsProcessed, state.TotalItems, formattedBytesProcessed)
t.term.SetStatus([]string{progress})
}
func (t *textPrinter) Error(item string, err error) error {
t.E("ignoring error for %s: %s\n", item, err)
return nil
}
func (t *textPrinter) CompleteItem(item string, size uint64, nodeType string) {
t.VV("dumped %s %v with size %v", nodeType, item, ui.FormatBytes(size))
}
func (t *textPrinter) Finish(state State, duration time.Duration) {
t.term.SetStatus(nil)
timeElapsed := ui.FormatDuration(duration)
formattedBytesProcessed := ui.FormatBytes(state.BytesProcessed)
summary := fmt.Sprintf("Summary: Dumped %d files, %d directories, %d total items (%s) in %s",
state.FilesProcessed, state.DirsProcessed, state.TotalItems, formattedBytesProcessed, timeElapsed)
t.term.Print(summary)
}

View file

@ -0,0 +1,50 @@
package dump
import (
"testing"
"time"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
)
func createTextProgress() (*ui.MockTerminal, ProgressPrinter) {
term := &ui.MockTerminal{}
printer := NewTextProgress(term, 3)
return term, printer
}
func TestTextPrintUpdate(t *testing.T) {
term, printer := createTextProgress()
printer.Update(State{10, 5, 15, 1000}, 5*time.Second)
test.Equals(t, []string{"[0:05] 10 files, 5 dirs, 15 total items 1000 B"}, term.Output)
}
func TestTextPrintSummary(t *testing.T) {
term, printer := createTextProgress()
printer.Finish(State{20, 10, 30, 2000}, 10*time.Second)
test.Equals(t, []string{"Summary: Dumped 20 files, 10 directories, 30 total items (1.953 KiB) in 0:10"}, term.Output)
}
func TestTextPrintCompleteItem(t *testing.T) {
for _, data := range []struct {
nodeType string
size uint64
expected string
}{
{"dir", 0, "dumped dir test with size 0 B"},
{"file", 123, "dumped file test with size 123 B"},
{"symlink", 0, "dumped symlink test with size 0 B"},
} {
term, printer := createTextProgress()
printer.CompleteItem("test", data.size, data.nodeType)
test.Equals(t, []string{data.expected}, term.Output)
}
}
func TestTextError(t *testing.T) {
term, printer := createTextProgress()
test.Equals(t, printer.Error("/path", errors.New("error \"message\"")), nil)
test.Equals(t, []string{"ignoring error for /path: error \"message\"\n"}, term.Errors)
}