diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5b745a608..44c47ede7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,10 +26,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Log in to the Container registry - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 396d04ee9..58590fd8c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,10 +48,6 @@ jobs: go: 1.24.x os: ubuntu-latest test_fuse: true - - job_name: Linux - go: 1.23.x - os: ubuntu-latest - test_fuse: true name: ${{ matrix.job_name }} Go ${{ matrix.go }} runs-on: ${{ matrix.os }} @@ -61,7 +57,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Go ${{ matrix.go }} uses: actions/setup-go@v6 @@ -224,7 +220,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Go ${{ env.latest_go }} uses: actions/setup-go@v6 @@ -246,7 +242,7 @@ jobs: checks: write steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Go ${{ env.latest_go }} uses: actions/setup-go@v6 @@ -254,7 +250,7 @@ jobs: go-version: ${{ env.latest_go }} - name: golangci-lint - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@v9 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. version: v2.4.0 @@ -291,7 +287,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Docker meta id: meta diff --git a/.golangci.yml b/.golangci.yml index 8538f4aee..fd703473f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -6,6 +6,8 @@ linters: - asciicheck # ensure that http response bodies are closed - bodyclose + # restrict imports from other restic packages for internal/backend (cache exempt) + - depguard - copyloopvar # make sure all errors returned by functions are handled - errcheck @@ -24,6 +26,20 @@ linters: # find unused variables, functions, structs, types, etc. - unused settings: + depguard: + rules: + # Prevent backend packages from importing the internal/restic package to keep the architectural layers intact. + backend-imports: + files: + - "**/internal/backend/**" + - "!**/internal/backend/cache/**" + - "!**/internal/backend/test/**" + - "!**/*_test.go" + deny: + - pkg: "github.com/restic/restic/internal/restic" + desc: "internal/restic should not be imported to keep the architectural layers intact" + - pkg: "github.com/restic/restic/internal/repository" + desc: "internal/repository should not be imported to keep the architectural layers intact" importas: alias: - pkg: github.com/restic/restic/internal/test diff --git a/build.go b/build.go index e6237bf21..94992dde4 100644 --- a/build.go +++ b/build.go @@ -36,7 +36,6 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //go:build ignore_build_go -// +build ignore_build_go package main @@ -60,7 +59,7 @@ var config = Config{ // see https://github.com/googleapis/google-cloud-go/issues/11448 DefaultBuildTags: []string{"selfupdate", "disable_grpc_modules"}, // specify build tags which are always used Tests: []string{"./..."}, // tests to run - MinVersion: GoVersion{Major: 1, Minor: 23, Patch: 0}, // minimum Go version supported + MinVersion: GoVersion{Major: 1, Minor: 24, Patch: 0}, // minimum Go version supported } // Config configures the build. diff --git a/changelog/unreleased/issue-3326 b/changelog/unreleased/issue-3326 new file mode 100644 index 000000000..c6246e489 --- /dev/null +++ b/changelog/unreleased/issue-3326 @@ -0,0 +1,9 @@ +Enhancement: `restic check` for specified snapshot(s) via snapshot filtering + +Snapshots can now be specified for the command `restic check` on the command line +via the standard snapshot filter, (`--tag`, `--host`, `--path` or specifying +snapshot IDs directly) and will be used for checking the packfiles used by these snapshots. + +https://github.com/restic/restic/issues/3326 +https://github.com/restic/restic/pull/5469 +https://github.com/restic/restic/pull/5644 diff --git a/changelog/unreleased/issue-4278 b/changelog/unreleased/issue-4278 new file mode 100644 index 000000000..5a88dbed9 --- /dev/null +++ b/changelog/unreleased/issue-4278 @@ -0,0 +1,12 @@ +Enhancement: Support include filters in `rewrite` command + +The enhancement enables the standard include filter options + --iinclude pattern same as --include pattern but ignores the casing of filenames + --iinclude-file file same as --include-file but ignores casing of filenames in patterns + -i, --include pattern include a pattern (can be specified multiple times) + --include-file file read include patterns from a file (can be specified multiple times) + +The exclusion or inclusion of filter parameters is exclusive, as in other commands. + +https://github.com/restic/restic/issues/4278 +https://github.com/restic/restic/pull/5191 diff --git a/changelog/unreleased/issue-5383 b/changelog/unreleased/issue-5383 index 9e79e6cf0..54778a181 100644 --- a/changelog/unreleased/issue-5383 +++ b/changelog/unreleased/issue-5383 @@ -7,3 +7,4 @@ anything in the terminal with certain terminal emulators. https://github.com/restic/restic/issues/5383 https://github.com/restic/restic/pull/5551 +https://github.com/restic/restic/pull/5626 diff --git a/changelog/unreleased/issue-5453 b/changelog/unreleased/issue-5453 new file mode 100644 index 000000000..12c09e0b0 --- /dev/null +++ b/changelog/unreleased/issue-5453 @@ -0,0 +1,10 @@ +Enhancement: `copy` copies snapshots in batches + +The `copy` command used to copy snapshots individually, even if this resulted in creating pack files +smaller than the target pack size. In particular, this resulted in many small files +when copying small incremental snapshots. + +Now, `copy` copies multiple snapshots at once to avoid creating small files. + +https://github.com/restic/restic/issues/5175 +https://github.com/restic/restic/pull/5464 diff --git a/changelog/unreleased/pull-4938 b/changelog/unreleased/pull-4938 new file mode 100644 index 000000000..26eae0b57 --- /dev/null +++ b/changelog/unreleased/pull-4938 @@ -0,0 +1,5 @@ +Change: Update dependencies and require Go 1.24 or newer + +We have updated all dependencies. Restic now requires Go 1.24 or newer to build. + +https://github.com/restic/restic/pull/5619 diff --git a/changelog/unreleased/pull-5465 b/changelog/unreleased/pull-5465 new file mode 100644 index 000000000..aee83881d --- /dev/null +++ b/changelog/unreleased/pull-5465 @@ -0,0 +1,10 @@ +Bugfix: Correctly restore ACL inheritance state on Windows + +Since the introduction of Security Descriptor backups in restic 0.17.0, the inheritance property of Access Control Entries (ACEs) was not restored correctly. This resulted in all restored permissions being marked as explicit (IsInherited: False), even if they were originally inherited from a parent folder. + +The issue was caused by sending conflicting inheritance flags (PROTECTED_... and UNPROTECTED_...) to the Windows API during the restore process. The API would default to the more restrictive PROTECTED state, effectively disabling inheritance. + +This has been fixed by ensuring that only the correct, non-conflicting inheritance flag is used when applying the security descriptor, preserving the original permission structure from the backup. + +https://github.com/restic/restic/pull/5465 +https://github.com/restic/restic/issues/5427 \ No newline at end of file diff --git a/changelog/unreleased/pull-5588 b/changelog/unreleased/pull-5588 new file mode 100644 index 000000000..dd3ab25e2 --- /dev/null +++ b/changelog/unreleased/pull-5588 @@ -0,0 +1,10 @@ +Enhancement: Display timezone information in snapshots output + +The `snapshots` command now displays which timezone is being used to show +timestamps. Since snapshots can be created in different timezones but are +always displayed in the local timezone, a footer line is now shown indicating +the timezone used for display (e.g., "Timestamps shown in CET timezone"). +This helps prevent confusion when comparing snapshots in a multi-user +environment. + +https://github.com/restic/restic/pull/5588 diff --git a/changelog/unreleased/pull-5592 b/changelog/unreleased/pull-5592 new file mode 100644 index 000000000..87a1bfd8b --- /dev/null +++ b/changelog/unreleased/pull-5592 @@ -0,0 +1,7 @@ +Bugfix: Return error if `RESTIC_PACK_SIZE` contains invalid value + +If the environment variable `RESTIC_PACK_SIZE` could not be parsed, then +restic ignored its value. Now, the restic commands fail with an error, unless +the command-line option `--pack-size` was specified. + +https://github.com/restic/restic/pull/5592 diff --git a/changelog/unreleased/pull-5610 b/changelog/unreleased/pull-5610 new file mode 100644 index 000000000..b6fd90821 --- /dev/null +++ b/changelog/unreleased/pull-5610 @@ -0,0 +1,7 @@ +Enhancement: reduce memory usage of check/copy/diff/stats commands + +We have optimized the memory usage of the `check`, `copy`, `diff` and +`stats` commands. These now require less memory when processing large +snapshots. + +https://github.com/restic/restic/pull/5610 diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index f93006f14..6ec24baf2 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/backend/cache" "github.com/restic/restic/internal/checker" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/global" "github.com/restic/restic/internal/repository" @@ -47,6 +48,7 @@ Exit status is 12 if the password is incorrect. GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { + finalizeSnapshotFilter(&opts.SnapshotFilter) summary, err := runCheck(cmd.Context(), opts, *globalOptions, args, globalOptions.Term) if globalOptions.JSON { if err != nil && summary.NumErrors == 0 { @@ -71,6 +73,7 @@ type CheckOptions struct { ReadDataSubset string CheckUnused bool WithCache bool + data.SnapshotFilter } func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) { @@ -84,6 +87,7 @@ func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) { panic(err) } f.BoolVar(&opts.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository") + initMultiSnapshotFilter(f, &opts.SnapshotFilter, true) } func checkFlags(opts CheckOptions) error { @@ -220,9 +224,6 @@ func prepareCheckCache(opts CheckOptions, gopts *global.Options, printer progres func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args []string, term ui.Terminal) (checkSummary, error) { summary := checkSummary{MessageType: "summary"} - if len(args) != 0 { - return summary, errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags") - } var printer progress.Printer if !gopts.JSON { @@ -231,11 +232,6 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args printer = newJSONErrorPrinter(term) } - readDataFilter, err := buildPacksFilter(opts, printer) - if err != nil { - return summary, err - } - cleanup := prepareCheckCache(opts, &gopts, printer) defer cleanup() @@ -249,7 +245,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args defer unlock() chkr := checker.New(repo, opts.CheckUnused) - err = chkr.LoadSnapshots(ctx) + err = chkr.LoadSnapshots(ctx, &opts.SnapshotFilter, args) if err != nil { return summary, err } @@ -365,6 +361,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args return summary, ctx.Err() } + // the following block only used for tests if opts.CheckUnused { unused, err := chkr.UnusedBlobs(ctx) if err != nil { @@ -376,6 +373,11 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args } } + readDataFilter, err := buildPacksFilter(opts, printer, chkr.IsFiltered()) + if err != nil { + return summary, err + } + if readDataFilter != nil { p := printer.NewCounter("packs") errChan := make(chan error) @@ -416,11 +418,16 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args return summary, nil } -func buildPacksFilter(opts CheckOptions, printer progress.Printer) (func(packs map[restic.ID]int64) map[restic.ID]int64, error) { +func buildPacksFilter(opts CheckOptions, printer progress.Printer, + filteredStatus bool) (func(packs map[restic.ID]int64) map[restic.ID]int64, error) { + typeData := "" + if filteredStatus { + typeData = "filtered " + } switch { case opts.ReadData: return func(packs map[restic.ID]int64) map[restic.ID]int64 { - printer.P("read all data\n") + printer.P("read all %sdata", typeData) return packs }, nil case opts.ReadDataSubset != "": @@ -431,7 +438,7 @@ func buildPacksFilter(opts CheckOptions, printer progress.Printer) (func(packs m return func(packs map[restic.ID]int64) map[restic.ID]int64 { packCount := uint64(len(packs)) packs = selectPacksByBucket(packs, bucket, totalBuckets) - printer.P("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, len(packs), packCount, totalBuckets) + printer.P("read group #%d of %d %sdata packs (out of total %d packs in %d groups", bucket, len(packs), typeData, packCount, totalBuckets) return packs }, nil } else if strings.HasSuffix(opts.ReadDataSubset, "%") { @@ -440,7 +447,7 @@ func buildPacksFilter(opts CheckOptions, printer progress.Printer) (func(packs m return nil, err } return func(packs map[restic.ID]int64) map[restic.ID]int64 { - printer.P("read %.1f%% of data packs\n", percentage) + printer.P("read %.1f%% of %spackfiles", percentage, typeData) return selectRandomPacksByPercentage(packs, percentage) }, nil } @@ -461,7 +468,7 @@ func buildPacksFilter(opts CheckOptions, printer progress.Printer) (func(packs m if repoSize == 0 { percentage = 100 } - printer.P("read %d bytes (%.1f%%) of data packs\n", subsetSize, percentage) + printer.P("read %d bytes (%.1f%%) of %sdata packs\n", subsetSize, percentage, typeData) return packs }, nil } diff --git a/cmd/restic/cmd_check_integration_test.go b/cmd/restic/cmd_check_integration_test.go index c4580100e..0a5bc3521 100644 --- a/cmd/restic/cmd_check_integration_test.go +++ b/cmd/restic/cmd_check_integration_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "strings" "testing" "github.com/restic/restic/internal/global" @@ -34,3 +35,67 @@ func testRunCheckOutput(t testing.TB, gopts global.Options, checkUnused bool) (s }) return buf.String(), err } + +func testRunCheckOutputWithOpts(t testing.TB, gopts global.Options, opts CheckOptions, args []string) (string, error) { + buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error { + gopts.Verbosity = 2 + _, err := runCheck(context.TODO(), opts, gopts, args, gopts.Term) + return err + }) + return buf.String(), err +} + +func TestCheckWithSnaphotFilter(t *testing.T) { + testCases := []struct { + opts CheckOptions + args []string + expectedOutput string + }{ + { // full --read-data, all snapshots + CheckOptions{ReadData: true}, + nil, + "4 / 4 packs", + }, + { // full --read-data, all snapshots + CheckOptions{ReadData: true}, + nil, + "2 / 2 snapshots", + }, + { // full --read-data, latest snapshot + CheckOptions{ReadData: true}, + []string{"latest"}, + "2 / 2 packs", + }, + { // full --read-data, latest snapshot + CheckOptions{ReadData: true}, + []string{"latest"}, + "1 / 1 snapshots", + }, + { // --read-data-subset, latest snapshot + CheckOptions{ReadDataSubset: "1%"}, + []string{"latest"}, + "1 / 1 packs", + }, + { // --read-data-subset, latest snapshot + CheckOptions{ReadDataSubset: "1%"}, + []string{"latest"}, + "filtered", + }, + } + + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{} + testRunBackup(t, env.testdata+"/0", []string{"for_cmd_ls"}, opts, env.gopts) + testRunBackup(t, env.testdata+"/0", []string{"0/9"}, opts, env.gopts) + + for _, testCase := range testCases { + output, err := testRunCheckOutputWithOpts(t, env.gopts, testCase.opts, testCase.args) + rtest.OK(t, err) + + hasOutput := strings.Contains(output, testCase.expectedOutput) + rtest.Assert(t, hasOutput, `expected to find substring %q, but did not find it`, testCase.expectedOutput) + } +} diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 498d6f75d..d17ded7c9 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -3,6 +3,9 @@ package main import ( "context" "fmt" + "iter" + "sync" + "time" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" @@ -12,7 +15,6 @@ import ( "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/progress" - "golang.org/x/sync/errgroup" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -70,6 +72,39 @@ func (opts *CopyOptions) AddFlags(f *pflag.FlagSet) { initMultiSnapshotFilter(f, &opts.SnapshotFilter, true) } +// collectAllSnapshots: select all snapshot trees to be copied +func collectAllSnapshots(ctx context.Context, opts CopyOptions, + srcSnapshotLister restic.Lister, srcRepo restic.Repository, + dstSnapshotByOriginal map[restic.ID][]*data.Snapshot, args []string, printer progress.Printer, +) iter.Seq[*data.Snapshot] { + return func(yield func(*data.Snapshot) bool) { + for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args, printer) { + // check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields + srcOriginal := *sn.ID() + if sn.Original != nil { + srcOriginal = *sn.Original + } + if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok { + isCopy := false + for _, originalSn := range originalSns { + if similarSnapshots(originalSn, sn) { + printer.V("\n%v", sn) + printer.V("skipping source snapshot %s, was already copied to snapshot %s", sn.ID().Str(), originalSn.ID().Str()) + isCopy = true + break + } + } + if isCopy { + continue + } + } + if !yield(sn) { + return + } + } + } +} + func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args []string, term ui.Terminal) error { printer := ui.NewProgressPrinter(false, gopts.Verbosity, term) secondaryGopts, isFromRepo, err := opts.SecondaryRepoOptions.FillGlobalOpts(ctx, gopts, "destination") @@ -124,49 +159,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args [ return ctx.Err() } - // remember already processed trees across all snapshots - visitedTrees := restic.NewIDSet() + selectedSnapshots := collectAllSnapshots(ctx, opts, srcSnapshotLister, srcRepo, dstSnapshotByOriginal, args, printer) - for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args, printer) { - // check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields - srcOriginal := *sn.ID() - if sn.Original != nil { - srcOriginal = *sn.Original - } - - if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok { - isCopy := false - for _, originalSn := range originalSns { - if similarSnapshots(originalSn, sn) { - printer.V("\n%v", sn) - printer.V("skipping source snapshot %s, was already copied to snapshot %s", sn.ID().Str(), originalSn.ID().Str()) - isCopy = true - break - } - } - if isCopy { - continue - } - } - printer.P("\n%v", sn) - printer.P(" copy started, this may take a while...") - if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, printer); err != nil { - return err - } - debug.Log("tree copied") - - // save snapshot - sn.Parent = nil // Parent does not have relevance in the new repo. - // Use Original as a persistent snapshot ID - if sn.Original == nil { - sn.Original = sn.ID() - } - newID, err := data.SaveSnapshot(ctx, dstRepo, sn) - if err != nil { - return err - } - printer.P("snapshot %s saved", newID.Str()) + if err := copyTreeBatched(ctx, srcRepo, dstRepo, selectedSnapshots, printer); err != nil { + return err } + return ctx.Err() } @@ -189,75 +187,136 @@ func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool { return true } -func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, - visitedTrees restic.IDSet, rootTreeID restic.ID, printer progress.Printer) error { +// copyTreeBatched copies multiple snapshots in one go. Snapshots are written after +// data equivalent to at least 10 packfiles was written. +func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, + selectedSnapshots iter.Seq[*data.Snapshot], printer progress.Printer) error { - wg, wgCtx := errgroup.WithContext(ctx) + // remember already processed trees across all snapshots + visitedTrees := srcRepo.NewAssociatedBlobSet() - treeStream := data.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool { - visited := visitedTrees.Has(treeID) - visitedTrees.Insert(treeID) - return visited - }, nil) + targetSize := uint64(dstRepo.PackSize()) * 100 + minDuration := 1 * time.Minute - copyBlobs := restic.NewBlobSet() - packList := restic.NewIDSet() + // use pull-based iterator to allow iteration in multiple steps + next, stop := iter.Pull(selectedSnapshots) + defer stop() - enqueue := func(h restic.BlobHandle) { - pb := srcRepo.LookupBlob(h.Type, h.ID) - copyBlobs.Insert(h) - for _, p := range pb { - packList.Insert(p.PackID) + for { + var batch []*data.Snapshot + batchSize := uint64(0) + startTime := time.Now() + + // call WithBlobUploader() once and then loop over all selectedSnapshots + err := dstRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + for batchSize < targetSize || time.Since(startTime) < minDuration { + sn, ok := next() + if !ok { + break + } + + batch = append(batch, sn) + + printer.P("\n%v", sn) + printer.P(" copy started, this may take a while...") + sizeBlobs, err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, printer, uploader) + if err != nil { + return err + } + debug.Log("tree copied") + batchSize += sizeBlobs + } + + return nil + }) + if err != nil { + return err + } + + // if no snapshots were processed in this batch, we're done + if len(batch) == 0 { + break + } + + // add a newline to separate saved snapshot messages from the other messages + if len(batch) > 1 { + printer.P("") + } + // save all the snapshots + for _, sn := range batch { + err := copySaveSnapshot(ctx, sn, dstRepo, printer) + if err != nil { + return err + } } } - wg.Go(func() error { - for tree := range treeStream { - if tree.Error != nil { - return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error) - } + return nil +} - // Do we already have this tree blob? - treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob} - if _, ok := dstRepo.LookupBlobSize(treeHandle.Type, treeHandle.ID); !ok { - // copy raw tree bytes to avoid problems if the serialization changes - enqueue(treeHandle) - } +func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, + visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaverWithAsync) (uint64, error) { - for _, entry := range tree.Nodes { - // Recursion into directories is handled by StreamTrees - // Copy the blobs for this file. - for _, blobID := range entry.Content { - h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID} - if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok { - enqueue(h) - } - } + copyBlobs := srcRepo.NewAssociatedBlobSet() + packList := restic.NewIDSet() + var lock sync.Mutex + + enqueue := func(h restic.BlobHandle) { + lock.Lock() + defer lock.Unlock() + if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok { + pb := srcRepo.LookupBlob(h.Type, h.ID) + copyBlobs.Insert(h) + for _, p := range pb { + packList.Insert(p.PackID) + } + } + } + + err := data.StreamTrees(ctx, srcRepo, restic.IDs{rootTreeID}, nil, func(treeID restic.ID) bool { + handle := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob} + visited := visitedTrees.Has(handle) + visitedTrees.Insert(handle) + return visited + }, func(treeID restic.ID, err error, nodes data.TreeNodeIterator) error { + if err != nil { + return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err) + } + + // copy raw tree bytes to avoid problems if the serialization changes + enqueue(restic.BlobHandle{ID: treeID, Type: restic.TreeBlob}) + + for item := range nodes { + if item.Error != nil { + return item.Error + } + // Recursion into directories is handled by StreamTrees + // Copy the blobs for this file. + for _, blobID := range item.Node.Content { + enqueue(restic.BlobHandle{Type: restic.DataBlob, ID: blobID}) } } return nil }) - err := wg.Wait() if err != nil { - return err + return 0, err } - copyStats(srcRepo, copyBlobs, packList, printer) + sizeBlobs := copyStats(srcRepo, copyBlobs, packList, printer) bar := printer.NewCounter("packs copied") - err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar, printer.P) + err = repository.CopyBlobs(ctx, srcRepo, dstRepo, uploader, packList, copyBlobs, bar, printer.P) if err != nil { - return errors.Fatalf("%s", err) + return 0, errors.Fatalf("%s", err) } - return nil + return sizeBlobs, nil } // copyStats: print statistics for the blobs to be copied -func copyStats(srcRepo restic.Repository, copyBlobs restic.BlobSet, packList restic.IDSet, printer progress.Printer) { - +func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, packList restic.IDSet, printer progress.Printer) uint64 { // count and size countBlobs := 0 sizeBlobs := uint64(0) - for blob := range copyBlobs { + for blob := range copyBlobs.Keys() { for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) { countBlobs++ sizeBlobs += uint64(blob.Length) @@ -267,4 +326,19 @@ func copyStats(srcRepo restic.Repository, copyBlobs restic.BlobSet, packList res printer.V(" copy %d blobs with disk size %s in %d packfiles\n", countBlobs, ui.FormatBytes(uint64(sizeBlobs)), len(packList)) + return sizeBlobs +} + +func copySaveSnapshot(ctx context.Context, sn *data.Snapshot, dstRepo restic.Repository, printer progress.Printer) error { + sn.Parent = nil // Parent does not have relevance in the new repo. + // Use Original as a persistent snapshot ID + if sn.Original == nil { + sn.Original = sn.ID() + } + newID, err := data.SaveSnapshot(ctx, dstRepo, sn) + if err != nil { + return err + } + printer.P("snapshot %s saved, copied from source snapshot %s", newID.Str(), sn.ID().Str()) + return nil } diff --git a/cmd/restic/cmd_copy_integration_test.go b/cmd/restic/cmd_copy_integration_test.go index c35e960ff..6105acfe4 100644 --- a/cmd/restic/cmd_copy_integration_test.go +++ b/cmd/restic/cmd_copy_integration_test.go @@ -7,7 +7,9 @@ import ( "testing" "github.com/restic/restic/internal/global" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui" ) func testRunCopy(t testing.TB, srcGopts global.Options, dstGopts global.Options) { @@ -83,6 +85,41 @@ func TestCopy(t *testing.T) { } rtest.Assert(t, len(origRestores) == 0, "found not copied snapshots") + + // check that snapshots were properly batched while copying + _, _, countBlobs := testPackAndBlobCounts(t, env.gopts) + countTreePacksDst, countDataPacksDst, countBlobsDst := testPackAndBlobCounts(t, env2.gopts) + + rtest.Equals(t, countBlobs, countBlobsDst, "expected blob count in boths repos to be equal") + rtest.Equals(t, countTreePacksDst, 1, "expected 1 tree packfile") + rtest.Equals(t, countDataPacksDst, 1, "expected 1 data packfile") +} + +func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks int, countDataPacks int, countBlobs int) { + rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error { + printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term) + _, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer) + rtest.OK(t, err) + defer unlock() + + rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { + blobs, _, err := repo.ListPack(context.TODO(), id, size) + rtest.OK(t, err) + rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob") + + switch blobs[0].Type { + case restic.TreeBlob: + countTreePacks++ + case restic.DataBlob: + countDataPacks++ + } + countBlobs += len(blobs) + return nil + })) + return nil + })) + + return countTreePacks, countDataPacks, countBlobs } func TestCopyIncremental(t *testing.T) { diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 48e6d58b7..d6e51107e 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -1,5 +1,4 @@ //go:build debug -// +build debug package main @@ -353,7 +352,7 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi return err } - err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { for _, blob := range list { printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length) if int(blob.Offset+blob.Length) > len(pack) { diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 3bd8e37b8..30b0878c0 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -5,7 +5,6 @@ import ( "encoding/json" "path" "reflect" - "sort" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" @@ -124,7 +123,7 @@ func (s *DiffStat) Add(node *data.Node) { } // addBlobs adds the blobs of node to s. -func addBlobs(bs restic.BlobSet, node *data.Node) { +func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) { if node == nil { return } @@ -148,18 +147,18 @@ func addBlobs(bs restic.BlobSet, node *data.Node) { } type DiffStatsContainer struct { - MessageType string `json:"message_type"` // "statistics" - SourceSnapshot string `json:"source_snapshot"` - TargetSnapshot string `json:"target_snapshot"` - ChangedFiles int `json:"changed_files"` - Added DiffStat `json:"added"` - Removed DiffStat `json:"removed"` - BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"` + MessageType string `json:"message_type"` // "statistics" + SourceSnapshot string `json:"source_snapshot"` + TargetSnapshot string `json:"target_snapshot"` + ChangedFiles int `json:"changed_files"` + Added DiffStat `json:"added"` + Removed DiffStat `json:"removed"` + BlobsBefore, BlobsAfter, BlobsCommon restic.AssociatedBlobSet `json:"-"` } // updateBlobs updates the blob counters in the stats struct. -func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat, printError func(string, ...interface{})) { - for h := range blobs { +func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *DiffStat, printError func(string, ...interface{})) { + for h := range blobs.Keys() { switch h.Type { case restic.DataBlob: stats.DataBlobs++ @@ -177,18 +176,21 @@ func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat, prin } } -func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error { +func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.AssociatedBlobSet, prefix string, id restic.ID) error { debug.Log("print %v tree %v", mode, id) tree, err := data.LoadTree(ctx, c.repo, id) if err != nil { return err } - for _, node := range tree.Nodes { + for item := range tree { + if item.Error != nil { + return item.Error + } if ctx.Err() != nil { return ctx.Err() } - + node := item.Node name := path.Join(prefix, node.Name) if node.Type == data.NodeTypeDir { name += "/" @@ -208,18 +210,22 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b return ctx.Err() } -func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error { +func (c *Comparer) collectDir(ctx context.Context, blobs restic.AssociatedBlobSet, id restic.ID) error { debug.Log("print tree %v", id) tree, err := data.LoadTree(ctx, c.repo, id) if err != nil { return err } - for _, node := range tree.Nodes { + for item := range tree { + if item.Error != nil { + return item.Error + } if ctx.Err() != nil { return ctx.Err() } + node := item.Node addBlobs(blobs, node) if node.Type == data.NodeTypeDir { @@ -233,29 +239,6 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id rest return ctx.Err() } -func uniqueNodeNames(tree1, tree2 *data.Tree) (tree1Nodes, tree2Nodes map[string]*data.Node, uniqueNames []string) { - names := make(map[string]struct{}) - tree1Nodes = make(map[string]*data.Node) - for _, node := range tree1.Nodes { - tree1Nodes[node.Name] = node - names[node.Name] = struct{}{} - } - - tree2Nodes = make(map[string]*data.Node) - for _, node := range tree2.Nodes { - tree2Nodes[node.Name] = node - names[node.Name] = struct{}{} - } - - uniqueNames = make([]string, 0, len(names)) - for name := range names { - uniqueNames = append(uniqueNames, name) - } - - sort.Strings(uniqueNames) - return tree1Nodes, tree2Nodes, uniqueNames -} - func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error { debug.Log("diffing %v to %v", id1, id2) tree1, err := data.LoadTree(ctx, c.repo, id1) @@ -268,21 +251,29 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref return err } - tree1Nodes, tree2Nodes, names := uniqueNodeNames(tree1, tree2) - - for _, name := range names { + for dt := range data.DualTreeIterator(tree1, tree2) { + if dt.Error != nil { + return dt.Error + } if ctx.Err() != nil { return ctx.Err() } - node1, t1 := tree1Nodes[name] - node2, t2 := tree2Nodes[name] + node1 := dt.Tree1 + node2 := dt.Tree2 + + var name string + if node1 != nil { + name = node1.Name + } else { + name = node2.Name + } addBlobs(stats.BlobsBefore, node1) addBlobs(stats.BlobsAfter, node2) switch { - case t1 && t2: + case node1 != nil && node2 != nil: name := path.Join(prefix, name) mod := "" @@ -328,7 +319,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref c.printError("error: %v", err) } } - case t1 && !t2: + case node1 != nil && node2 == nil: prefix := path.Join(prefix, name) if node1.Type == data.NodeTypeDir { prefix += "/" @@ -342,7 +333,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref c.printError("error: %v", err) } } - case !t1 && t2: + case node1 == nil && node2 != nil: prefix := path.Join(prefix, name) if node2.Type == data.NodeTypeDir { prefix += "/" @@ -442,9 +433,9 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [ MessageType: "statistics", SourceSnapshot: args[0], TargetSnapshot: args[1], - BlobsBefore: restic.NewBlobSet(), - BlobsAfter: restic.NewBlobSet(), - BlobsCommon: restic.NewBlobSet(), + BlobsBefore: repo.NewAssociatedBlobSet(), + BlobsAfter: repo.NewAssociatedBlobSet(), + BlobsCommon: repo.NewAssociatedBlobSet(), } stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree}) stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree}) diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index da45cc303..6a5e3adb6 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -80,7 +80,7 @@ func splitPath(p string) []string { return append(s, f) } -func printFromTree(ctx context.Context, tree *data.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error { +func printFromTree(ctx context.Context, tree data.TreeNodeIterator, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error { // If we print / we need to assume that there are multiple nodes at that // level in the tree. if pathComponents[0] == "" { @@ -92,11 +92,14 @@ func printFromTree(ctx context.Context, tree *data.Tree, repo restic.BlobLoader, item := filepath.Join(prefix, pathComponents[0]) l := len(pathComponents) - for _, node := range tree.Nodes { + for it := range tree { + if it.Error != nil { + return it.Error + } if ctx.Err() != nil { return ctx.Err() } - + node := it.Node // If dumping something in the highest level it will just take the // first item it finds and dump that according to the switch case below. if node.Name == pathComponents[0] { @@ -154,11 +157,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args [ } defer unlock() - sn, subfolder, err := (&data.SnapshotFilter{ - Hosts: opts.Hosts, - Paths: opts.Paths, - Tags: opts.Tags, - }).FindLatest(ctx, repo, repo, snapshotIDString) + sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, repo, repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 8e45c50a1..92e3abfcc 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -406,11 +406,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts global.Options, args []str } } - sn, subfolder, err := (&data.SnapshotFilter{ - Hosts: opts.Hosts, - Paths: opts.Paths, - Tags: opts.Tags, - }).FindLatest(ctx, snapshotLister, repo, args[0]) + sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, snapshotLister, repo, args[0]) if err != nil { return err } diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index a05f9c6b5..202109fa6 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package main diff --git a/cmd/restic/cmd_mount_disabled.go b/cmd/restic/cmd_mount_disabled.go index 4d45cbb6f..9ab1a7f52 100644 --- a/cmd/restic/cmd_mount_disabled.go +++ b/cmd/restic/cmd_mount_disabled.go @@ -1,5 +1,4 @@ //go:build !darwin && !freebsd && !linux -// +build !darwin,!freebsd,!linux package main diff --git a/cmd/restic/cmd_mount_integration_test.go b/cmd/restic/cmd_mount_integration_test.go index 197b21d52..a5f4a7aef 100644 --- a/cmd/restic/cmd_mount_integration_test.go +++ b/cmd/restic/cmd_mount_integration_test.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package main diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index ca22ee2de..bbb71972f 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -97,7 +97,11 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err continue } - for _, node := range tree.Nodes { + for item := range tree { + if item.Error != nil { + return item.Error + } + node := item.Node if node.Type == data.NodeTypeDir && node.Subtree != nil { trees[*node.Subtree] = true } @@ -134,28 +138,28 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err return ctx.Err() } - tree := data.NewTree(len(roots)) - for id := range roots { - var subtreeID = id - node := data.Node{ - Type: data.NodeTypeDir, - Name: id.Str(), - Mode: 0755, - Subtree: &subtreeID, - AccessTime: time.Now(), - ModTime: time.Now(), - ChangeTime: time.Now(), - } - err := tree.Insert(&node) - if err != nil { - return err - } - } - var treeID restic.ID - err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { var err error - treeID, err = data.SaveTree(ctx, uploader, tree) + tw := data.NewTreeWriter(uploader) + for id := range roots { + var subtreeID = id + node := data.Node{ + Type: data.NodeTypeDir, + Name: id.Str(), + Mode: 0755, + Subtree: &subtreeID, + AccessTime: time.Now(), + ModTime: time.Now(), + ChangeTime: time.Now(), + } + err := tw.AddNode(&node) + if err != nil { + return err + } + } + + treeID, err = tw.Finalize(ctx) if err != nil { return errors.Fatalf("unable to save new tree to the repository: %v", err) } diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 226da3d44..b392bf017 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -2,6 +2,7 @@ package main import ( "context" + "slices" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" @@ -130,7 +131,7 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp node.Size = newSize return node }, - RewriteFailedTree: func(_ restic.ID, path string, _ error) (*data.Tree, error) { + RewriteFailedTree: func(_ restic.ID, path string, _ error) (data.TreeNodeIterator, error) { if path == "/" { printer.P(" dir %q: not readable", path) // remove snapshots with invalid root node @@ -138,7 +139,7 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp } // If a subtree fails to load, remove it printer.P(" dir %q: replaced with empty directory", path) - return &data.Tree{}, nil + return slices.Values([]data.NodeOrError{}), nil }, AllowUnstableSerialization: true, }) @@ -150,7 +151,7 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp func(ctx context.Context, sn *data.Snapshot, uploader restic.BlobSaver) (restic.ID, *data.SnapshotSummary, error) { id, err := rewriter.RewriteTree(ctx, repo, uploader, "/", *sn.Tree) return id, nil, err - }, opts.DryRun, opts.Forget, nil, "repaired", printer) + }, opts.DryRun, opts.Forget, nil, "repaired", printer, false) if err != nil { return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 49b72c3f9..76a0044da 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -151,11 +151,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options, } defer unlock() - sn, subfolder, err := (&data.SnapshotFilter{ - Hosts: opts.Hosts, - Paths: opts.Paths, - Tags: opts.Tags, - }).FindLatest(ctx, repo, repo, snapshotIDString) + sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, repo, repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 76a504652..f5826c79f 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -30,6 +30,9 @@ The "rewrite" command excludes files from existing snapshots. It creates new snapshots containing the same data as the original ones, but without the files you specify to exclude. All metadata (time, host, tags) will be preserved. +Alternatively you can use one of the --include variants to only include files +in the new snapshot which you want to preserve. + The snapshots to rewrite are specified using the --host, --tag and --path options, or by providing a list of snapshot IDs. Please note that specifying neither any of these options nor a snapshot ID will cause the command to rewrite all snapshots. @@ -46,8 +49,8 @@ When rewrite is used with the --snapshot-summary option, a new snapshot is created containing statistics summary data. Only two fields in the summary will be non-zero: TotalFilesProcessed and TotalBytesProcessed. -When rewrite is called with one of the --exclude options, TotalFilesProcessed -and TotalBytesProcessed will be updated in the snapshot summary. +When rewrite is called with one of the --exclude or --include options, +TotalFilesProcessed and TotalBytesProcessed will be updated in the snapshot summary. EXIT STATUS =========== @@ -109,6 +112,7 @@ type RewriteOptions struct { Metadata snapshotMetadataArgs data.SnapshotFilter filter.ExcludePatternOptions + filter.IncludePatternOptions } func (opts *RewriteOptions) AddFlags(f *pflag.FlagSet) { @@ -120,6 +124,7 @@ func (opts *RewriteOptions) AddFlags(f *pflag.FlagSet) { initMultiSnapshotFilter(f, &opts.SnapshotFilter, true) opts.ExcludePatternOptions.Add(f) + opts.IncludePatternOptions.Add(f) } // rewriteFilterFunc returns the filtered tree ID or an error. If a snapshot summary is returned, the snapshot will @@ -136,33 +141,31 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *data. return false, err } + includeByNameFuncs, err := opts.IncludePatternOptions.CollectPatterns(printer.E) + if err != nil { + return false, err + } + metadata, err := opts.Metadata.convert() if err != nil { return false, err } + condInclude := len(includeByNameFuncs) > 0 + condExclude := len(rejectByNameFuncs) > 0 var filter rewriteFilterFunc - if len(rejectByNameFuncs) > 0 || opts.SnapshotSummary { - selectByName := func(nodepath string) bool { - for _, reject := range rejectByNameFuncs { - if reject(nodepath) { - return false - } - } - return true + if condInclude || condExclude || opts.SnapshotSummary { + var rewriteNode walker.NodeRewriteFunc + var keepEmptyDirectoryFunc walker.NodeKeepEmptyDirectoryFunc + if condInclude { + rewriteNode, keepEmptyDirectoryFunc = gatherIncludeFilters(includeByNameFuncs, printer) + } else { + rewriteNode = gatherExcludeFilters(rejectByNameFuncs, printer) } - rewriteNode := func(node *data.Node, path string) *data.Node { - if selectByName(path) { - return node - } - printer.P("excluding %s", path) - return nil - } - - rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode) + rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode, keepEmptyDirectoryFunc) filter = func(ctx context.Context, sn *data.Snapshot, uploader restic.BlobSaver) (restic.ID, *data.SnapshotSummary, error) { id, err := rewriter.RewriteTree(ctx, repo, uploader, "/", *sn.Tree) @@ -186,15 +189,16 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *data. } return filterAndReplaceSnapshot(ctx, repo, sn, - filter, opts.DryRun, opts.Forget, metadata, "rewrite", printer) + filter, opts.DryRun, opts.Forget, metadata, "rewrite", printer, len(includeByNameFuncs) > 0) } func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *data.Snapshot, - filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string, printer progress.Printer) (bool, error) { + filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string, printer progress.Printer, + keepEmptySnapshot bool) (bool, error) { var filteredTree restic.ID var summary *data.SnapshotSummary - err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { var err error filteredTree, summary, err = filter(ctx, sn, uploader) return err @@ -204,6 +208,10 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *d } if filteredTree.IsNull() { + if keepEmptySnapshot { + debug.Log("Snapshot %v not modified", sn) + return false, nil + } if dryRun { printer.P("would delete empty snapshot") } else { @@ -284,8 +292,12 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *d } func runRewrite(ctx context.Context, opts RewriteOptions, gopts global.Options, args []string, term ui.Terminal) error { - if !opts.SnapshotSummary && opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() { - return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided") + hasExcludes := !opts.ExcludePatternOptions.Empty() + hasIncludes := !opts.IncludePatternOptions.Empty() + if !opts.SnapshotSummary && !hasExcludes && !hasIncludes && opts.Metadata.empty() { + return errors.Fatal("Nothing to do: no excludes/includes provided and no new metadata provided") + } else if hasExcludes && hasIncludes { + return errors.Fatal("exclude and include patterns are mutually exclusive") } printer := ui.NewProgressPrinter(false, gopts.Verbosity, term) @@ -348,3 +360,72 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts global.Options, return nil } + +func gatherIncludeFilters(includeByNameFuncs []filter.IncludeByNameFunc, printer progress.Printer) (rewriteNode walker.NodeRewriteFunc, keepEmptyDirectory walker.NodeKeepEmptyDirectoryFunc) { + inSelectByName := func(nodepath string, node *data.Node) bool { + for _, include := range includeByNameFuncs { + matched, childMayMatch := include(nodepath) + if node.Type == data.NodeTypeDir { + // include directories if they or some of their children may be included + if matched || childMayMatch { + return true + } + } else if matched { + return true + } + } + return false + } + + rewriteNode = func(node *data.Node, path string) *data.Node { + if inSelectByName(path, node) { + if node.Type != data.NodeTypeDir { + printer.VV("including %q\n", path) + } + return node + } + return nil + } + + inSelectByNameDir := func(nodepath string) bool { + for _, include := range includeByNameFuncs { + matched, _ := include(nodepath) + if matched { + return matched + } + } + return false + } + + keepEmptyDirectory = func(path string) bool { + keep := inSelectByNameDir(path) + if keep { + printer.VV("including directory %q\n", path) + } + return keep + } + + return rewriteNode, keepEmptyDirectory +} + +func gatherExcludeFilters(excludeByNameFuncs []filter.RejectByNameFunc, printer progress.Printer) (rewriteNode walker.NodeRewriteFunc) { + exSelectByName := func(nodepath string) bool { + for _, reject := range excludeByNameFuncs { + if reject(nodepath) { + return false + } + } + return true + } + + rewriteNode = func(node *data.Node, path string) *data.Node { + if exSelectByName(path) { + return node + } + + printer.VV("excluding %q\n", path) + return nil + } + + return rewriteNode +} diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index 35f5f4d01..acef0dd4d 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -2,7 +2,9 @@ package main import ( "context" + "os" "path/filepath" + "strings" "testing" "github.com/restic/restic/internal/data" @@ -27,6 +29,27 @@ func testRunRewriteExclude(t testing.TB, gopts global.Options, excludes []string })) } +func testRunRewriteWithOpts(t testing.TB, opts RewriteOptions, gopts global.Options, args []string) error { + rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error { + return runRewrite(context.TODO(), opts, gopts, args, gopts.Term) + })) + return nil +} + +// testLsOutputContainsCount runs restic ls with the given options and asserts that +// exactly expectedCount lines of the output contain substring. +func testLsOutputContainsCount(t testing.TB, gopts global.Options, lsOpts LsOptions, lsArgs []string, substring string, expectedCount int) { + t.Helper() + out := testRunLsWithOpts(t, gopts, lsOpts, lsArgs) + count := 0 + for _, line := range strings.Split(string(out), "\n") { + if strings.Contains(line, substring) { + count++ + } + } + rtest.Assert(t, count == expectedCount, "expected %d lines containing %q, but got %d", expectedCount, substring, count) +} + func createBasicRewriteRepo(t testing.TB, env *testEnvironment) restic.ID { testSetupBackupData(t, env) @@ -39,6 +62,20 @@ func createBasicRewriteRepo(t testing.TB, env *testEnvironment) restic.ID { return snapshotIDs[0] } +func createBasicRewriteRepoWithEmptyDirectory(t testing.TB, env *testEnvironment) restic.ID { + testSetupBackupData(t, env) + + // make an empty directory named "empty-directory" + rtest.OK(t, os.Mkdir(filepath.Join(env.testdata, "/0/tests", "empty-directory"), 0755)) + + // create backup + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts) + snapshotIDs := testRunList(t, env.gopts, "snapshots") + rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs) + + return snapshotIDs[0] +} + func getSnapshot(t testing.TB, snapshotID restic.ID, env *testEnvironment) *data.Snapshot { t.Helper() @@ -195,3 +232,122 @@ func TestRewriteSnaphotSummary(t *testing.T) { rtest.Equals(t, oldSummary.TotalBytesProcessed, newSn.Summary.TotalBytesProcessed, "unexpected TotalBytesProcessed value") rtest.Equals(t, oldSummary.TotalFilesProcessed, newSn.Summary.TotalFilesProcessed, "unexpected TotalFilesProcessed value") } + +func TestRewriteInclude(t *testing.T) { + for _, tc := range []struct { + name string + opts RewriteOptions + lsSubstring string + lsExpectedCount int + summaryFilesExpected uint + }{ + {"relative", RewriteOptions{ + Forget: true, + IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"*.txt"}}, + }, ".txt", 2, 2}, + {"absolute", RewriteOptions{ + Forget: true, + // test that childMatches are working by only matching a subdirectory + IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"/testdata/0/for_cmd_ls"}}, + }, "/testdata/0", 5, 3}, + } { + t.Run(tc.name, func(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + createBasicRewriteRepo(t, env) + snapshots := testListSnapshots(t, env.gopts, 1) + + rtest.OK(t, testRunRewriteWithOpts(t, tc.opts, env.gopts, []string{"latest"})) + + newSnapshots := testListSnapshots(t, env.gopts, 1) + rtest.Assert(t, snapshots[0] != newSnapshots[0], "snapshot id should have changed") + + testLsOutputContainsCount(t, env.gopts, LsOptions{}, []string{"latest"}, tc.lsSubstring, tc.lsExpectedCount) + sn := testLoadSnapshot(t, env.gopts, newSnapshots[0]) + rtest.Assert(t, sn.Summary != nil, "snapshot should have a summary attached") + rtest.Assert(t, sn.Summary.TotalFilesProcessed == tc.summaryFilesExpected, + "there should be %d files in the snapshot, but there are %d files", tc.summaryFilesExpected, sn.Summary.TotalFilesProcessed) + }) + } +} + +func TestRewriteExcludeFiles(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + createBasicRewriteRepo(t, env) + snapshots := testListSnapshots(t, env.gopts, 1) + + // exclude txt files + err := testRunRewriteWithOpts(t, + RewriteOptions{ + Forget: true, + ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*.txt"}}, + }, + env.gopts, + []string{"latest"}) + rtest.OK(t, err) + newSnapshots := testListSnapshots(t, env.gopts, 1) + rtest.Assert(t, snapshots[0] != newSnapshots[0], "snapshot id should have changed") + + testLsOutputContainsCount(t, env.gopts, LsOptions{}, []string{"latest"}, ".txt", 0) +} + +func TestRewriteExcludeIncludeContradiction(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + testRunInit(t, env.gopts) + + // test contradiction + err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error { + return runRewrite(ctx, + RewriteOptions{ + ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"nonsense"}}, + IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"not allowed"}}, + }, + gopts, []string{"quack"}, env.gopts.Term) + }) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "exclude and include patterns are mutually exclusive"), `expected to fail command with message "exclude and include patterns are mutually exclusive"`) +} + +func TestRewriteIncludeEmptyDirectory(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + snapIDEmpty := createBasicRewriteRepoWithEmptyDirectory(t, env) + + // restic rewrite -i empty-directory --forget + // exclude txt files + err := testRunRewriteWithOpts(t, + RewriteOptions{ + Forget: true, + IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"empty-directory"}}, + }, + env.gopts, + []string{"latest"}) + rtest.OK(t, err) + newSnapshots := testListSnapshots(t, env.gopts, 1) + rtest.Assert(t, snapIDEmpty != newSnapshots[0], "snapshot id should have changed") + + testLsOutputContainsCount(t, env.gopts, LsOptions{}, []string{"latest"}, "empty-directory", 1) +} + +// TestRewriteIncludeNothing makes sure when nothing is included, the original snapshot stays untouched +func TestRewriteIncludeNothing(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + createBasicRewriteRepo(t, env) + snapsBefore := testListSnapshots(t, env.gopts, 1) + + // restic rewrite latest -i nothing-whatsoever --forget + err := testRunRewriteWithOpts(t, + RewriteOptions{ + Forget: true, + IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"nothing-whatsoever"}}, + }, + env.gopts, + []string{"latest"}) + rtest.OK(t, err) + + snapsAfter := testListSnapshots(t, env.gopts, 1) + rtest.Assert(t, snapsBefore[0] == snapsAfter[0], "snapshots should be identical but are %s and %s", + snapsBefore[0].Str(), snapsAfter[0].Str()) +} diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 059c92902..5f938fbe8 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -7,6 +7,7 @@ import ( "io" "sort" "strings" + "time" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/global" @@ -240,7 +241,15 @@ func PrintSnapshots(stdout io.Writer, list data.Snapshots, reasons []data.KeepRe tab.AddRow(data) } - tab.AddFooter(fmt.Sprintf("%d snapshots", len(list))) + // Add timezone information to prevent confusion: + // Each snapshot can be registered in different timezones, + // but we display them all in local timezone on this output. + footer := fmt.Sprintf("%d snapshots", len(list)) + zoneName, _ := time.Now().Local().Zone() + if zoneName != "" { + footer = fmt.Sprintf("Timestamps shown in %s timezone\n%s", zoneName, footer) + } + tab.AddFooter(footer) if multiline { // print an additional blank line between snapshots diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 2b7a74b94..490806e2a 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -130,7 +130,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args stats := &statsContainer{ uniqueFiles: make(map[fileID]struct{}), fileBlobs: make(map[string]restic.IDSet), - blobs: restic.NewBlobSet(), + blobs: repo.NewAssociatedBlobSet(), SnapshotsCount: 0, } @@ -146,7 +146,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args if opts.countMode == countModeRawData { // the blob handles have been collected, but not yet counted - for blobHandle := range stats.blobs { + for blobHandle := range stats.blobs.Keys() { pbs := repo.LookupBlob(blobHandle.Type, blobHandle.ID) if len(pbs) == 0 { return fmt.Errorf("blob %v not found", blobHandle) @@ -350,7 +350,7 @@ type statsContainer struct { // blobs is used to count individual unique blobs, // independent of references to files - blobs restic.BlobSet + blobs restic.AssociatedBlobSet } // fileID is a 256-bit hash that distinguishes unique files. diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go index c06fc8614..48f2e593d 100644 --- a/cmd/restic/cmd_unlock.go +++ b/cmd/restic/cmd_unlock.go @@ -19,6 +19,8 @@ func newUnlockCommand(globalOptions *global.Options) *cobra.Command { Long: ` The "unlock" command removes stale locks that have been created by other restic processes. +Removing locks works even with repositories served in append-only mode from restic's rest-server. + EXIT STATUS =========== diff --git a/cmd/restic/integration_helpers_unix_test.go b/cmd/restic/integration_helpers_unix_test.go index 30852a753..ffb2a9da6 100644 --- a/cmd/restic/integration_helpers_unix_test.go +++ b/cmd/restic/integration_helpers_unix_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package main diff --git a/cmd/restic/integration_helpers_windows_test.go b/cmd/restic/integration_helpers_windows_test.go index 42ced7e46..887810cde 100644 --- a/cmd/restic/integration_helpers_windows_test.go +++ b/cmd/restic/integration_helpers_windows_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package main diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 3307d2787..6ed2811f3 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -49,6 +49,10 @@ The full documentation can be found at https://restic.readthedocs.io/ . DisableAutoGenTag: true, PersistentPreRunE: func(c *cobra.Command, _ []string) error { + switch c.Name() { + case "__complete", "__completeNoDesc": + return nil + } return globalOptions.PreRun(needsPassword(c.Name())) }, } @@ -114,7 +118,7 @@ The full documentation can be found at https://restic.readthedocs.io/ . // user for authentication). func needsPassword(cmd string) bool { switch cmd { - case "cache", "generate", "help", "options", "self-update", "version", "__complete": + case "cache", "generate", "help", "options", "self-update", "version", "__complete", "__completeNoDesc": return false default: return true diff --git a/doc/020_installation.rst b/doc/020_installation.rst index f3c0b40b0..0b94c62ba 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -315,7 +315,7 @@ From Source *********** restic is written in the Go programming language and you need at least -Go version 1.23. Building restic may also work with older versions of Go, +Go version 1.24. Building restic may also work with older versions of Go, but that's not supported. See the `Getting started `__ guide of the Go project for instructions how to install Go. diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 334c5f94f..46cc30605 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -710,87 +710,4 @@ Environment Variables ********************* In addition to command-line options, restic supports passing various options in -environment variables. The following lists these environment variables: - -.. code-block:: console - - RESTIC_REPOSITORY_FILE Name of file containing the repository location (replaces --repository-file) - RESTIC_REPOSITORY Location of repository (replaces -r) - RESTIC_PASSWORD_FILE Location of password file (replaces --password-file) - RESTIC_PASSWORD The actual password for the repository - RESTIC_PASSWORD_COMMAND Command printing the password for the repository to stdout - RESTIC_KEY_HINT ID of key to try decrypting first, before other keys - RESTIC_CACERT Location(s) of certificate file(s), comma separated if multiple (replaces --cacert) - RESTIC_TLS_CLIENT_CERT Location of TLS client certificate and private key (replaces --tls-client-cert) - RESTIC_CACHE_DIR Location of the cache directory - RESTIC_COMPRESSION Compression mode (only available for repository format version 2) - RESTIC_HOST Only consider snapshots for this host / Set the hostname for the snapshot manually (replaces --host) - RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated - RESTIC_PACK_SIZE Target size for pack files - RESTIC_READ_CONCURRENCY Concurrency for file reads - - TMPDIR Location for temporary files (except Windows) - TMP Location for temporary files (only Windows) - - AWS_ACCESS_KEY_ID Amazon S3 access key ID - AWS_SECRET_ACCESS_KEY Amazon S3 secret access key - AWS_SESSION_TOKEN Amazon S3 temporary session token - AWS_DEFAULT_REGION Amazon S3 default region - AWS_PROFILE Amazon credentials profile (alternative to specifying key and region) - AWS_SHARED_CREDENTIALS_FILE Location of the AWS CLI shared credentials file (default: ~/.aws/credentials) - RESTIC_AWS_ASSUME_ROLE_ARN Amazon IAM Role ARN to assume using discovered credentials - RESTIC_AWS_ASSUME_ROLE_SESSION_NAME Session Name to use with the role assumption - RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID External ID to use with the role assumption - RESTIC_AWS_ASSUME_ROLE_POLICY Inline Amazion IAM session policy - RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption (default: us-east-1) - RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT URL to the STS endpoint (default is determined based on RESTIC_AWS_ASSUME_ROLE_REGION). You generally do not need to set this, advanced use only. - - AZURE_ACCOUNT_NAME Account name for Azure - AZURE_ACCOUNT_KEY Account key for Azure - AZURE_ACCOUNT_SAS Shared access signatures (SAS) for Azure - AZURE_ENDPOINT_SUFFIX Endpoint suffix for Azure Storage (default: core.windows.net) - AZURE_FORCE_CLI_CREDENTIAL Force the use of Azure CLI credentials for authentication - - B2_ACCOUNT_ID Account ID or applicationKeyId for Backblaze B2 - B2_ACCOUNT_KEY Account Key or applicationKey for Backblaze B2 - - GOOGLE_PROJECT_ID Project ID for Google Cloud Storage - GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json) - - OS_AUTH_URL Auth URL for keystone authentication - OS_REGION_NAME Region name for keystone authentication - OS_USERNAME Username for keystone authentication - OS_USER_ID User ID for keystone v3 authentication - OS_PASSWORD Password for keystone authentication - OS_TENANT_ID Tenant ID for keystone v2 authentication - OS_TENANT_NAME Tenant name for keystone v2 authentication - - OS_USER_DOMAIN_NAME User domain name for keystone authentication - OS_USER_DOMAIN_ID User domain ID for keystone v3 authentication - OS_PROJECT_NAME Project name for keystone authentication - OS_PROJECT_DOMAIN_NAME Project domain name for keystone authentication - OS_PROJECT_DOMAIN_ID Project domain ID for keystone v3 authentication - OS_TRUST_ID Trust ID for keystone v3 authentication - - OS_APPLICATION_CREDENTIAL_ID Application Credential ID (keystone v3) - OS_APPLICATION_CREDENTIAL_NAME Application Credential Name (keystone v3) - OS_APPLICATION_CREDENTIAL_SECRET Application Credential Secret (keystone v3) - - OS_STORAGE_URL Storage URL for token authentication - OS_AUTH_TOKEN Auth token for token authentication - - RCLONE_BWLIMIT rclone bandwidth limit - - RESTIC_REST_USERNAME Restic REST Server username - RESTIC_REST_PASSWORD Restic REST Server password - - ST_AUTH Auth URL for keystone v1 authentication - ST_USER Username for keystone v1 authentication - ST_KEY Password for keystone v1 authentication - -See :ref:`caching` for the rules concerning cache locations when -``RESTIC_CACHE_DIR`` is not set. - -The external programs that restic may execute include ``rclone`` (for rclone -backends) and ``ssh`` (for the SFTP backend). These may respond to further -environment variables and configuration files; see their respective manuals. +environment variables. See :ref:`environment-variables` for a list. diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 75a7e79f1..ba263fd31 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -205,21 +205,28 @@ example from a local to a remote repository, you can use the ``copy`` command: .. code-block:: console - $ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo + $ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo --verbose repository d6504c63 opened successfully repository 3dd0878c opened successfully + [0:00] 100.00% 2 / 2 index files loaded + [0:00] 100.00% 7 / 7 index files loaded snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST by user@kasimir copy started, this may take a while... - snapshot 7a746a07 saved + [0:00] 100.00% 13 / 13 packs copied snapshot 4e5d5487 of [/home/user/work] at 2020-05-01 22:44:07.012113 +0200 CEST by user@kasimir skipping snapshot 4e5d5487, was already copied to snapshot 50eb62b7 + snapshot 7a746a07 saved, copied from source snapshot 410b18a2 + The example command copies all snapshots from the source repository ``/srv/restic-repo`` to the destination repository ``/srv/restic-repo-copy``. Snapshots which have previously been copied between repositories will -be skipped by later copy runs. +be skipped by later copy runs. Information about skipped snapshots is only +printed when ``--verbose`` is passed to the command. For efficiency reasons, +the snapshots are copied in batches, such that the ``snapshot [...] saved`` +messages can appear some time after the snapshot content was copied. .. important:: This process will have to both download (read) and upload (write) the entire snapshot(s) due to the different encryption keys used in the @@ -329,6 +336,13 @@ The options ``--exclude``, ``--exclude-file``, ``--iexclude`` and ``--iexclude-file`` are supported. They behave the same way as for the backup command, see :ref:`backup-excluding-files` for details. +The options ``--include``, ``--include-file``, ``--iinclude`` and +``--iinclude-file`` are supported as well. +The ``--include`` variants allow you to reduce an existing snapshot or a set of snapshots +to those files that you are really interested in. An example could be all pictures +from a snapshot: +``restic rewrite -r ... --iinclude "*.jpg" --iinclude "*.jpeg" --iinclude "*.png"``. + It is possible to rewrite only a subset of snapshots by filtering them the same way as for the ``copy`` command, see :ref:`copy-filtering-snapshots`. @@ -353,7 +367,7 @@ modifying the repository. Instead restic will only print the actions it would perform. .. note:: The ``rewrite`` command verifies that it does not modify snapshots in - unexpected ways and fails with an ``cannot encode tree at "[...]" without losing information`` + unexpected ways and fails with an ``cannot encode tree at "[...]" without loosing information`` error otherwise. This can occur when rewriting a snapshot created by a newer version of restic or some third-party implementation. diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_parameters.rst similarity index 92% rename from doc/047_tuning_backup_parameters.rst rename to doc/047_tuning_parameters.rst index f1b89a7eb..eba958211 100644 --- a/doc/047_tuning_backup_parameters.rst +++ b/doc/047_tuning_parameters.rst @@ -9,14 +9,14 @@ ^ for subsubsections " for paragraphs -######################## -Tuning Backup Parameters -######################## +################# +Tuning Parameters +################# -Restic offers a few parameters that allow tuning the backup. The default values should -work well in general although specific use cases can benefit from different non-default -values. As the restic commands evolve over time, the optimal value for each parameter -can also change across restic versions. +Restic offers a few parameters that allow tuning the backup and other operations. +The default values should work well in general although specific use cases can +benefit from different non-default values. As the restic commands evolve over +time, the optimal value for each parameter can also change across restic versions. Disabling Backup Progress Estimation @@ -99,7 +99,8 @@ Swift and some Google Drive Team accounts, where there are hard limits on the to number of files. Larger pack sizes can also improve the backup speed for a repository stored on a local HDD. This can be achieved by either using the ``--pack-size`` option or defining the ``$RESTIC_PACK_SIZE`` environment variable, using an integer value for the -pack size in MiB. Restic currently defaults to a 16 MiB pack size. +pack size in MiB. Restic currently defaults to a 16 MiB pack size. Note that this +setting should be specified for each restic command that modifies the repository. The side effect of increasing the pack size is requiring more disk space for temporary pack files created before uploading. The space must be available in the system default temp diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 5d9de34b6..a24e597dd 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -14,8 +14,99 @@ Scripting ######################### -This is a list of how certain tasks may be accomplished when you use -restic via scripts. +This section covers environment variables and how certain tasks may be accomplished +when you use restic via scripts. + +.. _environment-variables: + +Environment Variables +********************* + +In addition to command-line options, restic supports passing various options in +environment variables, which are listed below. + +.. code-block:: console + + RESTIC_REPOSITORY_FILE Name of file containing the repository location (replaces --repository-file) + RESTIC_REPOSITORY Location of repository (replaces -r) + RESTIC_PASSWORD_FILE Location of password file (replaces --password-file) + RESTIC_PASSWORD The actual password for the repository + RESTIC_PASSWORD_COMMAND Command printing the password for the repository to stdout + RESTIC_KEY_HINT ID of key to try decrypting first, before other keys + RESTIC_CACERT Location(s) of certificate file(s), comma separated if multiple (replaces --cacert) + RESTIC_TLS_CLIENT_CERT Location of TLS client certificate and private key (replaces --tls-client-cert) + RESTIC_CACHE_DIR Location of the cache directory + RESTIC_COMPRESSION Compression mode (only available for repository format version 2) + RESTIC_HOST Only consider snapshots for this host / Set the hostname for the snapshot manually (replaces --host) + RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated + RESTIC_PACK_SIZE Target size for pack files + RESTIC_READ_CONCURRENCY Concurrency for file reads + + TMPDIR Location for temporary files (except Windows) + TMP Location for temporary files (only Windows) + + AWS_ACCESS_KEY_ID Amazon S3 access key ID + AWS_SECRET_ACCESS_KEY Amazon S3 secret access key + AWS_SESSION_TOKEN Amazon S3 temporary session token + AWS_DEFAULT_REGION Amazon S3 default region + AWS_PROFILE Amazon credentials profile (alternative to specifying key and region) + AWS_SHARED_CREDENTIALS_FILE Location of the AWS CLI shared credentials file (default: ~/.aws/credentials) + RESTIC_AWS_ASSUME_ROLE_ARN Amazon IAM Role ARN to assume using discovered credentials + RESTIC_AWS_ASSUME_ROLE_SESSION_NAME Session Name to use with the role assumption + RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID External ID to use with the role assumption + RESTIC_AWS_ASSUME_ROLE_POLICY Inline Amazion IAM session policy + RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption (default: us-east-1) + RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT URL to the STS endpoint (default is determined based on RESTIC_AWS_ASSUME_ROLE_REGION). You generally do not need to set this, advanced use only. + + AZURE_ACCOUNT_NAME Account name for Azure + AZURE_ACCOUNT_KEY Account key for Azure + AZURE_ACCOUNT_SAS Shared access signatures (SAS) for Azure + AZURE_ENDPOINT_SUFFIX Endpoint suffix for Azure Storage (default: core.windows.net) + AZURE_FORCE_CLI_CREDENTIAL Force the use of Azure CLI credentials for authentication + + B2_ACCOUNT_ID Account ID or applicationKeyId for Backblaze B2 + B2_ACCOUNT_KEY Account Key or applicationKey for Backblaze B2 + + GOOGLE_PROJECT_ID Project ID for Google Cloud Storage + GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json) + + OS_AUTH_URL Auth URL for keystone authentication + OS_REGION_NAME Region name for keystone authentication + OS_USERNAME Username for keystone authentication + OS_USER_ID User ID for keystone v3 authentication + OS_PASSWORD Password for keystone authentication + OS_TENANT_ID Tenant ID for keystone v2 authentication + OS_TENANT_NAME Tenant name for keystone v2 authentication + + OS_USER_DOMAIN_NAME User domain name for keystone authentication + OS_USER_DOMAIN_ID User domain ID for keystone v3 authentication + OS_PROJECT_NAME Project name for keystone authentication + OS_PROJECT_DOMAIN_NAME Project domain name for keystone authentication + OS_PROJECT_DOMAIN_ID Project domain ID for keystone v3 authentication + OS_TRUST_ID Trust ID for keystone v3 authentication + + OS_APPLICATION_CREDENTIAL_ID Application Credential ID (keystone v3) + OS_APPLICATION_CREDENTIAL_NAME Application Credential Name (keystone v3) + OS_APPLICATION_CREDENTIAL_SECRET Application Credential Secret (keystone v3) + + OS_STORAGE_URL Storage URL for token authentication + OS_AUTH_TOKEN Auth token for token authentication + + RCLONE_BWLIMIT rclone bandwidth limit + + RESTIC_REST_USERNAME Restic REST Server username + RESTIC_REST_PASSWORD Restic REST Server password + + ST_AUTH Auth URL for keystone v1 authentication + ST_USER Username for keystone v1 authentication + ST_KEY Password for keystone v1 authentication + +See :ref:`caching` for the rules concerning cache locations when +``RESTIC_CACHE_DIR`` is not set. + +The external programs that restic may execute include ``rclone`` (for rclone +backends) and ``ssh`` (for the SFTP backend). These may respond to further +environment variables and configuration files; see their respective manuals. Check if a repository is already initialized ******************************************** @@ -810,11 +901,11 @@ Changed Summary ^^^^^^^ -+----------------------------+-----------------------------------+--------+ -| ``message_type`` | Always "summary" | string | -+----------------------------+-----------------------------------+--------+ -| ``changed_snapshot_count`` | Total number of changed snapshots | int64 | -+----------------------------+-----------------------------------+--------+ ++-----------------------+-----------------------------------+--------+ +| ``message_type`` | Always "summary" | string | ++-----------------------+-----------------------------------+--------+ +| ``changed_snapshots`` | Total number of changed snapshots | int64 | ++-----------------------+-----------------------------------+--------+ version ------- diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 36c9d63ec..fd19f121d 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -82,6 +82,12 @@ If ``check`` detects damaged pack files, it will show instructions on how to rep them using the ``repair pack`` command. Use that command instead of the "Repair the index" section in this guide. +If you are interested to check only specific snapshots, you can now +use the standard snapshot filter method specifying ``--host``, ``--path``, ``--tag`` or +alternatively naming snapshot ID(s) explicitely. The selected subset of packfiles +will then be checked for consistency and read when either ``--read-data`` or +``--read-data-subset`` is given. + 2. Backup the repository ************************ diff --git a/doc/100_references.rst b/doc/100_references.rst index 4be4e0dae..a5a3f10ff 100644 --- a/doc/100_references.rst +++ b/doc/100_references.rst @@ -19,5 +19,6 @@ Design ****** .. include:: design.rst +.. include:: view_repository.rst .. include:: cache.rst .. include:: REST_backend.rst diff --git a/doc/_static/css/restic.css b/doc/_static/css/restic.css index a4cf25421..67ce38164 100644 --- a/doc/_static/css/restic.css +++ b/doc/_static/css/restic.css @@ -8,3 +8,7 @@ height: 50% !important; width: 50% !important; } + +.wy-table-responsive table td { + white-space: normal; +} diff --git a/doc/index.rst b/doc/index.rst index 8b72dcf58..ecfdabaa8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -9,7 +9,7 @@ Restic Documentation 030_preparing_a_new_repo 040_backup 045_working_with_repos - 047_tuning_backup_parameters + 047_tuning_parameters 050_restore 060_forget 070_encryption diff --git a/doc/view_repository.rst b/doc/view_repository.rst new file mode 100644 index 000000000..280446cd9 --- /dev/null +++ b/doc/view_repository.rst @@ -0,0 +1,137 @@ +.. + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s + Style Guide for documenting which you may follow: + + # with overline, for parts + * for chapters + = for sections + - for subsections + ^ for subsubsections + " for paragraphs + + +************************ +Diving into a Repository +************************ +The following section dives into the commands developers could use +to extract certain data from a repository. + +Listing different file types in the repository +============================================== + +The ``restic list`` allows listing objects in the repository based on type. +The allowed types are (in alphabetic order): + +- blobs +- index +- keys +- locks +- packs +- snapshots + +With the exception of ``blobs`` all output - in text mode - contains zero or more +``IDs`` of the given type, one ``ID`` per output line. + +The output for ``blobs`` contains one or more lines of output of the form +``blob-type blob-ID``, where ``blob-type`` is either ``data`` or ``tree``, and ``blob-ID`` +is the ``sha256sum`` of the ``blob``. + +The output of the ``restic list 'type-plural'`` is most commonly used for the ``restic cat 'type' ID`` +command to study an ``type`` object with an ``ID`` in more detail. The only exception to +this singular/plural ``type`` is ``ìndex``, which is used in both commands ``restic list index`` and +``restic cat index ``. + +The examples below are using part of the standard file structure for testing restic itself. +Here is the ``ls`` output of the one and only snapshot in this test repository: + +.. code-block:: console + + $ restic -r /srv/restic-repo ls 4254d65c + snapshot 4254d65c of [/srv/restic-repo/testdata/0/for_cmd_ls] at 2026-01-17 17:26:41.972899252 +0000 UTC by user@kasimir filtered by []: + /srv/restic-repo/testdata + /srv/restic-repo/testdata/0 + /srv/restic-repo/testdata/0/for_cmd_ls + /srv/restic-repo/testdata/0/for_cmd_ls/file1.txt + /srv/restic-repo/testdata/0/for_cmd_ls/file2.txt + /srv/restic-repo/testdata/0/for_cmd_ls/python.py + +Inspecting this repository with ``restic list snapshots`` produces: + +.. code-block:: console + + $ restic -r /srv/restic-repo list snapshots -q + 4254d65c92208eda22b852b390bd5401ca4c500be7a022c70e7c33de68ca2143 + $ restic -r /srv/restic-repo cat snapshot 4254d65c92208eda22b852b390bd5401ca4c500be7a022c70e7c33de68ca2143 -q + { + "time": "2026-01-17T17:26:41.972899252Z", + "tree": "db9e90f7f1761ab892b3ae25e3838bbd697499b985e9b47d3a1da09e0bd8ca68", + ... + "summary": { + "backup_start": "2026-01-17T17:26:41.972899252Z", + "backup_end": "2026-01-17T17:26:42.581012438Z", + ... + } + } + +The index contains 2 packfiles, one for trees and one for the actual file data: + +.. code-block:: console + + $ restic -r /srv/restic-repo list index -q + a1828d209e760f0fd143aa79e530de0a377d7affd1cd0964d9cb2ad3c77e0d8b + $ restic -r /srv/restic-repo cat index a1828d209e760f0fd143aa79e530de0a377d7affd1cd0964d9cb2ad3c77e0d8b | jq + { + "packs": [ + { + "id": "953e5381138bdc44da23740a83065809dd4021f45ce4e351b577dc4c07f81314", + "blobs": [ + { + "id": "124323c57d74fb8944c98fb69ce67a41a107cb6d2ed304cf50c8529cc137aafd", + "type": "data", + "offset": 0, + "length": 59, + "uncompressed_length": 18 + }, + ... + ] + }, + { + "id": "75bca8556f47d16362e58e757ea89a34b28fb96aedcc314bea35d468e5cb665c", + "blobs": [ + { + "id": "6dfdc53cc3b45a6bf519a7fb80a54f6ef3e3ea859f51d3e85a6235177606f1f9", + "type": "tree", + "offset": 0, + "length": 271, + "uncompressed_length": 353 + }, + ... + ] + } + ] + } + +And this is the list of blobs: + +.. code-block:: console + + $ restic -r /srv/restic-repo list blobs -q + data 124323c57d74fb8944c98fb69ce67a41a107cb6d2ed304cf50c8529cc137aafd + data 37cc0b45af245d93abaecba73a600a8d577b39e4a1fdc2dcdf93ad63b1e167bd + data 5dfb8bc8a35175bf011d10ac7bc3a6b8d42b7743ac188be8c1bf0b215f9b7bf5 + tree 6dfdc53cc3b45a6bf519a7fb80a54f6ef3e3ea859f51d3e85a6235177606f1f9 + tree 73947e98d4025179347363401eb41f148dc29a1d1735bfb96a08a6036422108c + tree 6d1daddbb3f280be0f25e708618576e003c2a87516a9aa31e98205ae0a152ab5 + tree 2e89c815e31c377629ef77fa1c156d1ad794b9f09d9d3b113e00e8eab36ceb98 + tree db9e90f7f1761ab892b3ae25e3838bbd697499b985e9b47d3a1da09e0bd8ca68 + tree d2524f3358bffbfe7349ca73df4bd3f23f5b252a9ba887481eda7e696b506dd4 + tree 4d8f5a6c6e90a2d69ae4b2f8e4f7f5851ccc4fa2cd3314f81de6c929453994fe + +The other types ``keys``, ``locks`` and ``packs`` are used in the same way as the type ``index``. + +.. code-block:: console + + $ restic -r /srv/restic-repo list packs -q + 953e5381138bdc44da23740a83065809dd4021f45ce4e351b577dc4c07f81314 + 75bca8556f47d16362e58e757ea89a34b28fb96aedcc314bea35d468e5cb665c diff --git a/go.mod b/go.mod index bd4022ae5..f6e683181 100644 --- a/go.mod +++ b/go.mod @@ -1,61 +1,61 @@ module github.com/restic/restic -go 1.23.0 +go 1.24.0 // keep the old behavior for reparse points on windows until handling reparse points has been improved in restic // https://forum.restic.net/t/windows-junction-backup-with-go1-23-or-later/8940 godebug winsymlink=0 require ( - cloud.google.com/go/storage v1.56.1 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 + cloud.google.com/go/storage v1.59.2 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/Backblaze/blazer v0.7.2 github.com/Microsoft/go-winio v0.6.2 github.com/anacrolix/fuse v0.3.1 github.com/cenkalti/backoff/v4 v4.3.0 github.com/cespare/xxhash/v2 v2.3.0 - github.com/elithrar/simple-scrypt v1.3.0 + github.com/elithrar/simple-scrypt v1.4.0 github.com/go-ole/go-ole v1.3.0 github.com/google/go-cmp v0.7.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/klauspost/compress v1.18.0 - github.com/minio/minio-go/v7 v7.0.95 - github.com/ncw/swift/v2 v2.0.4 + github.com/klauspost/compress v1.18.3 + github.com/minio/minio-go/v7 v7.0.98 + github.com/ncw/swift/v2 v2.0.5 github.com/peterbourgon/unixtransport v0.0.7 github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.7.0 github.com/pkg/sftp v1.13.10 github.com/pkg/xattr v0.4.12 github.com/restic/chunker v0.4.0 - github.com/spf13/cobra v1.10.1 + github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 go.uber.org/automaxprocs v1.6.0 - golang.org/x/crypto v0.41.0 - golang.org/x/net v0.43.0 - golang.org/x/oauth2 v0.30.0 - golang.org/x/sync v0.16.0 - golang.org/x/sys v0.35.0 - golang.org/x/term v0.34.0 - golang.org/x/text v0.28.0 - golang.org/x/time v0.12.0 - google.golang.org/api v0.248.0 + golang.org/x/crypto v0.47.0 + golang.org/x/net v0.49.0 + golang.org/x/oauth2 v0.34.0 + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.40.0 + golang.org/x/term v0.39.0 + golang.org/x/text v0.33.0 + golang.org/x/time v0.14.0 + google.golang.org/api v0.256.0 ) require ( cel.dev/expr v0.24.0 // indirect - cloud.google.com/go v0.121.6 // indirect - cloud.google.com/go/auth v0.16.5 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.17.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/compute/metadata v0.8.0 // indirect - cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect cloud.google.com/go/monitoring v1.24.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 // indirect github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -64,21 +64,21 @@ require ( github.com/felixge/fgprof v0.9.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-jose/go-jose/v4 v4.1.2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/goccy/go-json v0.10.5 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect @@ -86,21 +86,21 @@ require ( github.com/rs/xid v1.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect - github.com/tinylib/msgp v1.3.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/zeebo/errs v1.4.0 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/sdk v1.36.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.36.0 // indirect - google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect - google.golang.org/grpc v1.74.2 // indirect - google.golang.org/protobuf v1.36.7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect + google.golang.org/grpc v1.76.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect ) diff --git a/go.sum b/go.sum index cd2cc7dbd..59509e3ec 100644 --- a/go.sum +++ b/go.sum @@ -1,51 +1,51 @@ cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= -cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= -cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= -cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI= -cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= +cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA= -cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw= -cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= -cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= -cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= -cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/longrunning v0.7.0 h1:FV0+SYF1RIj59gyoWDRi45GiYUMM3K1qO51qoboQT1E= +cloud.google.com/go/longrunning v0.7.0/go.mod h1:ySn2yXmjbK9Ba0zsQqunhDkYi0+9rlXIwnoAf+h+TPY= cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= -cloud.google.com/go/storage v1.56.1 h1:n6gy+yLnHn0hTwBFzNn8zJ1kqWfR91wzdM8hjRF4wP0= -cloud.google.com/go/storage v1.56.1/go.mod h1:C9xuCZgFl3buo2HZU/1FncgvvOgTAs/rnh4gF4lMg0s= +cloud.google.com/go/storage v1.59.2 h1:gmOAuG1opU8YvycMNpP+DvHfT9BfzzK5Cy+arP+Nocw= +cloud.google.com/go/storage v1.59.2/go.mod h1:cMWbtM+anpC74gn6qjLh+exqYcfmB9Hqe5z6adx+CLI= cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 h1:KpMC6LFL7mqpExyMC9jVOYRiVhLmamjeZfRsUpB7l4s= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0/go.mod h1:J7MUC/wtRpfGVbQ5sIItY5/FuVWmvzlY21WAOfQnq/I= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI= -github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQM= github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 h1:lhhYARPUu3LmHysQ/igznQphfzynnqI3D75oUyw1HXk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0/go.mod h1:l9rva3ApbBpEJxSNYnwT9N4CDLrWgtq3u8736C5hyJw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.54.0 h1:xfK3bbi6F2RDtaZFtUdKO3osOBIhNb+xTs8lFW6yx9o= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.54.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 h1:s0WlVbf9qpvkh1c/uDAPElam0WrL7fHRIidgZJ7UqZI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -78,8 +78,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvyukov/go-fuzz v0.0.0-20220726122315-1d375ef9f9f6/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= -github.com/elithrar/simple-scrypt v1.3.0 h1:KIlOlxdoQf9JWKl5lMAJ28SY2URB0XTRDn2TckyzAZg= -github.com/elithrar/simple-scrypt v1.3.0/go.mod h1:U2XQRI95XHY0St410VE3UjT7vuKb1qPwrl/EJwEqnZo= +github.com/elithrar/simple-scrypt v1.4.0 h1:5sZ4st4bK5wBymv7DaGZFbRasbAj0WrhxECkLSm6q6Y= +github.com/elithrar/simple-scrypt v1.4.0/go.mod h1:dtDmWT+ijC9sdIbf50kQjgI9xOFmA4LG/8/T6TbmtFY= github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= @@ -95,8 +95,8 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI= +github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -104,8 +104,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -122,8 +120,8 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= -github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -133,11 +131,13 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -151,14 +151,14 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= -github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= -github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= -github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= -github.com/ncw/swift/v2 v2.0.4 h1:hHWVFxn5/YaTWAASmn4qyq2p6OyP/Hm3vMLzkjEqR7w= -github.com/ncw/swift/v2 v2.0.4/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= +github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8= +github.com/ncw/swift/v2 v2.0.5/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/ff/v3 v3.3.1/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= @@ -190,14 +190,14 @@ github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -214,41 +214,43 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= -github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4= golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= @@ -261,16 +263,16 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -284,22 +286,22 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= @@ -308,18 +310,20 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.248.0 h1:hUotakSkcwGdYUqzCRc5yGYsg4wXxpkKlW5ryVqvC1Y= -google.golang.org/api v0.248.0/go.mod h1:yAFUAF56Li7IuIQbTFoLwXTCI6XCFKueOlS7S9e4F9k= -google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= -google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= -google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ= -google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= -google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= -google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.256.0 h1:u6Khm8+F9sxbCTYNoBHg6/Hwv0N/i+V94MvkOSor6oI= +google.golang.org/api v0.256.0/go.mod h1:KIgPhksXADEKJlnEoRa9qAII4rXcy40vfI8HRqcU964= +google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc= +google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc= +google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba h1:B14OtaXuMaCQsl2deSvNkyPKIzq3BjfxQp8d00QyWx4= +google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G5IanEx8/PgI9w6CFcYQf7jMtHQhZruvfM1i3qOqk5U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index b1711b557..319dfe01f 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -96,7 +96,6 @@ type Archiver struct { FS fs.FS Options Options - blobSaver *blobSaver fileSaver *fileSaver treeSaver *treeSaver mu sync.Mutex @@ -145,11 +144,6 @@ type Options struct { // turned out to be a good default for most situations). ReadConcurrency uint - // SaveBlobConcurrency sets how many blobs are hashed and saved - // concurrently. If it's set to zero, the default is the number of CPUs - // available in the system. - SaveBlobConcurrency uint - // SaveTreeConcurrency sets how many trees are marshalled and saved to the // repo concurrently. SaveTreeConcurrency uint @@ -165,12 +159,6 @@ func (o Options) ApplyDefaults() Options { o.ReadConcurrency = 2 } - if o.SaveBlobConcurrency == 0 { - // blob saving is CPU bound due to hash checking and encryption - // the actual upload is handled by the repository itself - o.SaveBlobConcurrency = uint(runtime.GOMAXPROCS(0)) - } - if o.SaveTreeConcurrency == 0 { // can either wait for a file, wait for a tree, serialize a tree or wait for saveblob // the last two are cpu-bound and thus mutually exclusive. @@ -293,7 +281,7 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig // loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned. // If there is no node to load, then nil is returned without an error. -func (arch *Archiver) loadSubtree(ctx context.Context, node *data.Node) (*data.Tree, error) { +func (arch *Archiver) loadSubtree(ctx context.Context, node *data.Node) (data.TreeNodeIterator, error) { if node == nil || node.Type != data.NodeTypeDir || node.Subtree == nil { return nil, nil } @@ -319,7 +307,7 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { // saveDir stores a directory in the repo and returns the node. snPath is the // path within the current snapshot. -func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous *data.Tree, complete fileCompleteFunc) (d futureNode, err error) { +func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous data.TreeNodeIterator, complete fileCompleteFunc) (d futureNode, err error) { debug.Log("%v %v", snPath, dir) treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta) @@ -329,6 +317,9 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me nodes := make([]futureNode, 0, len(names)) + finder := data.NewTreeFinder(previous) + defer finder.Close() + for _, name := range names { // test if context has been cancelled if ctx.Err() != nil { @@ -337,7 +328,11 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me } pathname := arch.FS.Join(dir, name) - oldNode := previous.Find(name) + oldNode, err := finder.Find(name) + err = arch.error(pathname, err) + if err != nil { + return futureNode{}, err + } snItem := join(snPath, name) fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode) @@ -657,7 +652,7 @@ func join(elem ...string) string { // saveTree stores a Tree in the repo, returned is the tree. snPath is the path // within the current snapshot. -func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous *data.Tree, complete fileCompleteFunc) (futureNode, int, error) { +func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous data.TreeNodeIterator, complete fileCompleteFunc) (futureNode, int, error) { var node *data.Node if snPath != "/" { @@ -675,10 +670,13 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, node = &data.Node{} } - debug.Log("%v (%v nodes), parent %v", snPath, len(atree.Nodes), previous) + debug.Log("%v (%v nodes)", snPath, len(atree.Nodes)) nodeNames := atree.NodeNames() nodes := make([]futureNode, 0, len(nodeNames)) + finder := data.NewTreeFinder(previous) + defer finder.Close() + // iterate over the nodes of atree in lexicographic (=deterministic) order for _, name := range nodeNames { subatree := atree.Nodes[name] @@ -690,7 +688,13 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, // this is a leaf node if subatree.Leaf() { - fn, excluded, err := arch.save(ctx, join(snPath, name), subatree.Path, previous.Find(name)) + pathname := join(snPath, name) + oldNode, err := finder.Find(name) + err = arch.error(pathname, err) + if err != nil { + return futureNode{}, 0, err + } + fn, excluded, err := arch.save(ctx, pathname, subatree.Path, oldNode) if err != nil { err = arch.error(subatree.Path, err) @@ -710,7 +714,11 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, snItem := join(snPath, name) + "/" start := time.Now() - oldNode := previous.Find(name) + oldNode, err := finder.Find(name) + err = arch.error(snItem, err) + if err != nil { + return futureNode{}, 0, err + } oldSubtree, err := arch.loadSubtree(ctx, oldNode) if err != nil { err = arch.error(join(snPath, name), err) @@ -842,7 +850,7 @@ type SnapshotOptions struct { } // loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned. -func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) *data.Tree { +func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) data.TreeNodeIterator { if sn == nil { return nil } @@ -863,24 +871,20 @@ func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) *da } // runWorkers starts the worker pools, which are stopped when the context is cancelled. -func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaver) { - arch.blobSaver = newBlobSaver(ctx, wg, uploader, arch.Options.SaveBlobConcurrency) - +func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync) { arch.fileSaver = newFileSaver(ctx, wg, - arch.blobSaver.Save, + uploader, arch.Repo.Config().ChunkerPolynomial, - arch.Options.ReadConcurrency, arch.Options.SaveBlobConcurrency) + arch.Options.ReadConcurrency) arch.fileSaver.CompleteBlob = arch.CompleteBlob arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo - arch.treeSaver = newTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.blobSaver.Save, arch.Error) + arch.treeSaver = newTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, uploader, arch.Error) } func (arch *Archiver) stopWorkers() { - arch.blobSaver.TriggerShutdown() arch.fileSaver.TriggerShutdown() arch.treeSaver.TriggerShutdown() - arch.blobSaver = nil arch.fileSaver = nil arch.treeSaver = nil } @@ -903,7 +907,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps var rootTreeID restic.ID - err = arch.Repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + err = arch.Repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, wgCtx := errgroup.WithContext(ctx) start := time.Now() diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index 1c20c4b76..d92ac7fcf 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -56,7 +56,7 @@ func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS return err } - err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) arch.runWorkers(ctx, wg, uploader) @@ -219,7 +219,7 @@ func TestArchiverSave(t *testing.T) { arch.summary = &Summary{} var fnr futureNodeResult - err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) arch.runWorkers(ctx, wg, uploader) @@ -296,7 +296,7 @@ func TestArchiverSaveReaderFS(t *testing.T) { arch.summary = &Summary{} var fnr futureNodeResult - err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) arch.runWorkers(ctx, wg, uploader) @@ -415,29 +415,39 @@ type blobCountingRepo struct { saved map[restic.BlobHandle]uint } -func (repo *blobCountingRepo) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaver) error) error { - return repo.archiverRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { +func (repo *blobCountingRepo) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaverWithAsync) error) error { + return repo.archiverRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { return fn(ctx, &blobCountingSaver{saver: uploader, blobCountingRepo: repo}) }) } type blobCountingSaver struct { - saver restic.BlobSaver + saver restic.BlobSaverWithAsync blobCountingRepo *blobCountingRepo } +func (repo *blobCountingSaver) count(exists bool, h restic.BlobHandle) { + if exists { + return + } + repo.blobCountingRepo.m.Lock() + repo.blobCountingRepo.saved[h]++ + repo.blobCountingRepo.m.Unlock() +} + func (repo *blobCountingSaver) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) { id, exists, size, err := repo.saver.SaveBlob(ctx, t, buf, id, storeDuplicate) - if exists { - return id, exists, size, err - } - h := restic.BlobHandle{ID: id, Type: t} - repo.blobCountingRepo.m.Lock() - repo.blobCountingRepo.saved[h]++ - repo.blobCountingRepo.m.Unlock() + repo.count(exists, restic.BlobHandle{ID: id, Type: t}) return id, exists, size, err } +func (repo *blobCountingSaver) SaveBlobAsync(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool, cb func(newID restic.ID, known bool, size int, err error)) { + repo.saver.SaveBlobAsync(ctx, t, buf, id, storeDuplicate, func(newID restic.ID, known bool, size int, err error) { + repo.count(known, restic.BlobHandle{ID: newID, Type: t}) + cb(newID, known, size, err) + }) +} + func appendToFile(t testing.TB, filename string, data []byte) { f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { @@ -840,7 +850,7 @@ func TestArchiverSaveDir(t *testing.T) { defer back() var treeID restic.ID - err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) arch.runWorkers(ctx, wg, uploader) meta, err := testFS.OpenFile(test.target, fs.O_NOFOLLOW, true) @@ -867,11 +877,7 @@ func TestArchiverSaveDir(t *testing.T) { } node.Name = targetNodeName - tree := &data.Tree{Nodes: []*data.Node{node}} - treeID, err = data.SaveTree(ctx, uploader, tree) - if err != nil { - t.Fatal(err) - } + treeID = data.TestSaveNodes(t, ctx, uploader, []*data.Node{node}) arch.stopWorkers() return wg.Wait() }) @@ -906,7 +912,7 @@ func TestArchiverSaveDirIncremental(t *testing.T) { arch.summary = &Summary{} var fnr futureNodeResult - err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) arch.runWorkers(ctx, wg, uploader) meta, err := testFS.OpenFile(tempdir, fs.O_NOFOLLOW, true) @@ -1096,7 +1102,7 @@ func TestArchiverSaveTree(t *testing.T) { } var treeID restic.ID - err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) arch.runWorkers(ctx, wg, uploader) @@ -2077,8 +2083,6 @@ func TestArchiverContextCanceled(t *testing.T) { type TrackFS struct { fs.FS - errorOn map[string]error - opened map[string]uint m sync.Mutex } @@ -2094,38 +2098,61 @@ func (m *TrackFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, e type failSaveRepo struct { archiverRepo failAfter int32 - cnt int32 + cnt atomic.Int32 err error } -func (f *failSaveRepo) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaver) error) error { - return f.archiverRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { - return fn(ctx, &failSaveSaver{saver: uploader, failSaveRepo: f}) +func (f *failSaveRepo) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaverWithAsync) error) error { + outerCtx, outerCancel := context.WithCancelCause(ctx) + defer outerCancel(f.err) + return f.archiverRepo.WithBlobUploader(outerCtx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + return fn(ctx, &failSaveSaver{saver: uploader, failSaveRepo: f, semaphore: make(chan struct{}, 1), outerCancel: outerCancel}) }) } type failSaveSaver struct { - saver restic.BlobSaver + saver restic.BlobSaverWithAsync failSaveRepo *failSaveRepo + semaphore chan struct{} + outerCancel context.CancelCauseFunc } func (f *failSaveSaver) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) { - val := atomic.AddInt32(&f.failSaveRepo.cnt, 1) + val := f.failSaveRepo.cnt.Add(1) if val >= f.failSaveRepo.failAfter { - return restic.Hash(buf), false, 0, f.failSaveRepo.err + return restic.ID{}, false, 0, f.failSaveRepo.err } return f.saver.SaveBlob(ctx, t, buf, id, storeDuplicate) } -func TestArchiverAbortEarlyOnError(t *testing.T) { - var testErr = errors.New("test error") +func (f *failSaveSaver) SaveBlobAsync(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool, cb func(newID restic.ID, known bool, size int, err error)) { + // limit concurrency to make test reliable + f.semaphore <- struct{}{} + val := f.failSaveRepo.cnt.Add(1) + if val >= f.failSaveRepo.failAfter { + // kill the outer context to make SaveBlobAsync fail + // precisely injecting a specific error into the repository is not possible, so just cancel the context + f.outerCancel(f.failSaveRepo.err) + } + + f.saver.SaveBlobAsync(ctx, t, buf, id, storeDuplicate, func(newID restic.ID, known bool, size int, err error) { + if val >= f.failSaveRepo.failAfter { + if err == nil { + panic("expected error") + } + } + cb(newID, known, size, err) + <-f.semaphore + }) +} + +func TestArchiverAbortEarlyOnError(t *testing.T) { var tests = []struct { src TestDir wantOpen map[string]uint failAfter uint // error after so many blobs have been saved to the repo - err error }{ { src: TestDir{ @@ -2137,8 +2164,6 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { }, wantOpen: map[string]uint{ filepath.FromSlash("dir/bar"): 1, - filepath.FromSlash("dir/baz"): 1, - filepath.FromSlash("dir/foo"): 1, }, }, { @@ -2165,9 +2190,8 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { filepath.FromSlash("dir/file9"): 0, }, // fails after four to seven files were opened, as the ReadConcurrency allows for - // two queued files and SaveBlobConcurrency for one blob queued for saving. + // two queued files and one blob queued for saving. failAfter: 4, - err: testErr, }, } @@ -2186,25 +2210,25 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { opened: make(map[string]uint), } - if testFS.errorOn == nil { - testFS.errorOn = make(map[string]error) - } - + testErr := context.Canceled testRepo := &failSaveRepo{ archiverRepo: repo, failAfter: int32(test.failAfter), - err: test.err, + err: testErr, } // at most two files may be queued arch := New(testRepo, testFS, Options{ - ReadConcurrency: 2, - SaveBlobConcurrency: 1, + ReadConcurrency: 2, }) + arch.Error = func(item string, err error) error { + t.Logf("archiver error for %q: %v", item, err) + return err + } _, _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) - if !errors.Is(err, test.err) { - t.Errorf("expected error (%v) not found, got %v", test.err, err) + if !errors.Is(err, testErr) { + t.Errorf("expected error (%v) not found, got %v", testErr, err) } t.Logf("Snapshot return error: %v", err) @@ -2231,19 +2255,15 @@ func snapshot(t testing.TB, repo archiverRepo, fs fs.FS, parent *data.Snapshot, ParentSnapshot: parent, } snapshot, _, _, err := arch.Snapshot(ctx, []string{filename}, sopts) - if err != nil { - t.Fatal(err) - } - + rtest.OK(t, err) tree, err := data.LoadTree(ctx, repo, *snapshot.Tree) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) - node := tree.Find(filename) - if node == nil { - t.Fatalf("unable to find node for testfile in snapshot") - } + finder := data.NewTreeFinder(tree) + defer finder.Close() + node, err := finder.Find(filename) + rtest.OK(t, err) + rtest.Assert(t, node != nil, "unable to find node for testfile in snapshot") return snapshot, node } @@ -2428,7 +2448,7 @@ func TestRacyFileTypeSwap(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _ = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + _ = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx) arch := New(repo, fs.Track{FS: statfs}, Options{}) diff --git a/internal/archiver/archiver_unix_test.go b/internal/archiver/archiver_unix_test.go index 55f677cd8..978dc1b80 100644 --- a/internal/archiver/archiver_unix_test.go +++ b/internal/archiver/archiver_unix_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package archiver diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go deleted file mode 100644 index 356a32ce2..000000000 --- a/internal/archiver/blob_saver.go +++ /dev/null @@ -1,105 +0,0 @@ -package archiver - -import ( - "context" - "fmt" - - "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" - "golang.org/x/sync/errgroup" -) - -// saver allows saving a blob. -type saver interface { - SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) -} - -// blobSaver concurrently saves incoming blobs to the repo. -type blobSaver struct { - repo saver - ch chan<- saveBlobJob -} - -// newBlobSaver returns a new blob. A worker pool is started, it is stopped -// when ctx is cancelled. -func newBlobSaver(ctx context.Context, wg *errgroup.Group, repo saver, workers uint) *blobSaver { - ch := make(chan saveBlobJob) - s := &blobSaver{ - repo: repo, - ch: ch, - } - - for i := uint(0); i < workers; i++ { - wg.Go(func() error { - return s.worker(ctx, ch) - }) - } - - return s -} - -func (s *blobSaver) TriggerShutdown() { - close(s.ch) -} - -// Save stores a blob in the repo. It checks the index and the known blobs -// before saving anything. It takes ownership of the buffer passed in. -func (s *blobSaver) Save(ctx context.Context, t restic.BlobType, buf *buffer, filename string, cb func(res saveBlobResponse)) { - select { - case s.ch <- saveBlobJob{BlobType: t, buf: buf, fn: filename, cb: cb}: - case <-ctx.Done(): - debug.Log("not sending job, context is cancelled") - } -} - -type saveBlobJob struct { - restic.BlobType - buf *buffer - fn string - cb func(res saveBlobResponse) -} - -type saveBlobResponse struct { - id restic.ID - length int - sizeInRepo int - known bool -} - -func (s *blobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) { - id, known, sizeInRepo, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) - - if err != nil { - return saveBlobResponse{}, err - } - - return saveBlobResponse{ - id: id, - length: len(buf), - sizeInRepo: sizeInRepo, - known: known, - }, nil -} - -func (s *blobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { - for { - var job saveBlobJob - var ok bool - select { - case <-ctx.Done(): - return nil - case job, ok = <-jobs: - if !ok { - return nil - } - } - - res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data) - if err != nil { - debug.Log("saveBlob returned error, exiting: %v", err) - return fmt.Errorf("failed to save blob from file %q: %w", job.fn, err) - } - job.cb(res) - job.buf.Release() - } -} diff --git a/internal/archiver/blob_saver_test.go b/internal/archiver/blob_saver_test.go deleted file mode 100644 index e23ed12e5..000000000 --- a/internal/archiver/blob_saver_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package archiver - -import ( - "context" - "fmt" - "runtime" - "strings" - "sync" - "sync/atomic" - "testing" - - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" - rtest "github.com/restic/restic/internal/test" - "golang.org/x/sync/errgroup" -) - -var errTest = errors.New("test error") - -type saveFail struct { - cnt int32 - failAt int32 -} - -func (b *saveFail) SaveBlob(_ context.Context, _ restic.BlobType, _ []byte, id restic.ID, _ bool) (restic.ID, bool, int, error) { - val := atomic.AddInt32(&b.cnt, 1) - if val == b.failAt { - return restic.ID{}, false, 0, errTest - } - - return id, false, 0, nil -} - -func TestBlobSaver(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - wg, ctx := errgroup.WithContext(ctx) - saver := &saveFail{} - - b := newBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) - - var wait sync.WaitGroup - var results []saveBlobResponse - var lock sync.Mutex - - wait.Add(20) - for i := 0; i < 20; i++ { - buf := &buffer{Data: []byte(fmt.Sprintf("foo%d", i))} - idx := i - lock.Lock() - results = append(results, saveBlobResponse{}) - lock.Unlock() - b.Save(ctx, restic.DataBlob, buf, "file", func(res saveBlobResponse) { - lock.Lock() - results[idx] = res - lock.Unlock() - wait.Done() - }) - } - - wait.Wait() - for i, sbr := range results { - if sbr.known { - t.Errorf("blob %v is known, that should not be the case", i) - } - } - - b.TriggerShutdown() - - err := wg.Wait() - if err != nil { - t.Fatal(err) - } -} - -func TestBlobSaverError(t *testing.T) { - var tests = []struct { - blobs int - failAt int - }{ - {20, 2}, - {20, 5}, - {20, 15}, - {200, 150}, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - wg, ctx := errgroup.WithContext(ctx) - saver := &saveFail{ - failAt: int32(test.failAt), - } - - b := newBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) - - for i := 0; i < test.blobs; i++ { - buf := &buffer{Data: []byte(fmt.Sprintf("foo%d", i))} - b.Save(ctx, restic.DataBlob, buf, "errfile", func(res saveBlobResponse) {}) - } - - b.TriggerShutdown() - - err := wg.Wait() - if err == nil { - t.Errorf("expected error not found") - } - - rtest.Assert(t, errors.Is(err, errTest), "unexpected error %v", err) - rtest.Assert(t, strings.Contains(err.Error(), "errfile"), "expected error to contain 'errfile' got: %v", err) - }) - } -} diff --git a/internal/archiver/buffer.go b/internal/archiver/buffer.go index d5bfb46b3..0a6ae6d8f 100644 --- a/internal/archiver/buffer.go +++ b/internal/archiver/buffer.go @@ -1,5 +1,7 @@ package archiver +import "sync" + // buffer is a reusable buffer. After the buffer has been used, Release should // be called so the underlying slice is put back into the pool. type buffer struct { @@ -14,41 +16,32 @@ func (b *buffer) Release() { return } - select { - case pool.ch <- b: - default: - } + pool.pool.Put(b) } // bufferPool implements a limited set of reusable buffers. type bufferPool struct { - ch chan *buffer + pool sync.Pool defaultSize int } // newBufferPool initializes a new buffer pool. The pool stores at most max // items. New buffers are created with defaultSize. Buffers that have grown // larger are not put back. -func newBufferPool(max int, defaultSize int) *bufferPool { +func newBufferPool(defaultSize int) *bufferPool { b := &bufferPool{ - ch: make(chan *buffer, max), defaultSize: defaultSize, } + b.pool = sync.Pool{New: func() any { + return &buffer{ + Data: make([]byte, defaultSize), + pool: b, + } + }} return b } // Get returns a new buffer, either from the pool or newly allocated. func (pool *bufferPool) Get() *buffer { - select { - case buf := <-pool.ch: - return buf - default: - } - - b := &buffer{ - Data: make([]byte, pool.defaultSize), - pool: pool, - } - - return b + return pool.pool.Get().(*buffer) } diff --git a/internal/archiver/buffer_test.go b/internal/archiver/buffer_test.go new file mode 100644 index 000000000..1b577fa2d --- /dev/null +++ b/internal/archiver/buffer_test.go @@ -0,0 +1,58 @@ +package archiver + +import ( + "testing" +) + +func TestBufferPoolReuse(t *testing.T) { + success := false + // retries to avoid flakiness. The test can fail depending on the GC. + for i := 0; i < 100; i++ { + // Test that buffers are actually reused from the pool + pool := newBufferPool(1024) + + // Get a buffer and modify it + buf1 := pool.Get() + buf1.Data[0] = 0xFF + originalAddr := &buf1.Data[0] + buf1.Release() + + // Get another buffer and check if it's the same underlying slice + buf2 := pool.Get() + if &buf2.Data[0] == originalAddr { + success = true + break + } + buf2.Release() + } + if !success { + t.Error("buffer was not reused from pool") + } +} + +func TestBufferPoolLargeBuffers(t *testing.T) { + success := false + // retries to avoid flakiness. The test can fail depending on the GC. + for i := 0; i < 100; i++ { + // Test that buffers larger than defaultSize are not returned to pool + pool := newBufferPool(1024) + buf := pool.Get() + + // Grow the buffer beyond default size + buf.Data = append(buf.Data, make([]byte, 2048)...) + originalCap := cap(buf.Data) + + buf.Release() + + // Get a new buffer - should not be the same slice + newBuf := pool.Get() + if cap(newBuf.Data) != originalCap { + success = true + break + } + } + + if !success { + t.Error("large buffer was incorrectly returned to pool") + } +} diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 8370bee4d..84e175d82 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -15,13 +15,10 @@ import ( "golang.org/x/sync/errgroup" ) -// saveBlobFn saves a blob to a repo. -type saveBlobFn func(context.Context, restic.BlobType, *buffer, string, func(res saveBlobResponse)) - // fileSaver concurrently saves incoming files to the repo. type fileSaver struct { saveFilePool *bufferPool - saveBlob saveBlobFn + uploader restic.BlobSaverAsync pol chunker.Pol @@ -34,16 +31,13 @@ type fileSaver struct { // newFileSaver returns a new file saver. A worker pool with fileWorkers is // started, it is stopped when ctx is cancelled. -func newFileSaver(ctx context.Context, wg *errgroup.Group, save saveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint) *fileSaver { +func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, pol chunker.Pol, fileWorkers uint) *fileSaver { ch := make(chan saveFileJob) - - debug.Log("new file saver with %v file workers and %v blob workers", fileWorkers, blobWorkers) - - poolSize := fileWorkers + blobWorkers + debug.Log("new file saver with %v file workers", fileWorkers) s := &fileSaver{ - saveBlob: save, - saveFilePool: newBufferPool(int(poolSize), chunker.MaxSize), + uploader: uploader, + saveFilePool: newBufferPool(chunker.MaxSize), pol: pol, ch: ch, @@ -203,15 +197,20 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat node.Content = append(node.Content, restic.ID{}) lock.Unlock() - s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr saveBlobResponse) { - lock.Lock() - if !sbr.known { - fnr.stats.DataBlobs++ - fnr.stats.DataSize += uint64(sbr.length) - fnr.stats.DataSizeInRepo += uint64(sbr.sizeInRepo) + s.uploader.SaveBlobAsync(ctx, restic.DataBlob, buf.Data, restic.ID{}, false, func(newID restic.ID, known bool, sizeInRepo int, err error) { + defer buf.Release() + if err != nil { + completeError(err) + return } - node.Content[pos] = sbr.id + lock.Lock() + if !known { + fnr.stats.DataBlobs++ + fnr.stats.DataSize += uint64(len(buf.Data)) + fnr.stats.DataSizeInRepo += uint64(sizeInRepo) + } + node.Content[pos] = newID lock.Unlock() completeBlob() diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index 5aab78558..4dbf78548 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -11,7 +11,6 @@ import ( "github.com/restic/chunker" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) @@ -31,44 +30,35 @@ func createTestFiles(t testing.TB, num int) (files []string) { return files } -func startFileSaver(ctx context.Context, t testing.TB, _ fs.FS) (*fileSaver, context.Context, *errgroup.Group) { +func startFileSaver(ctx context.Context, t testing.TB, _ fs.FS) (*fileSaver, *mockSaver, context.Context, *errgroup.Group) { wg, ctx := errgroup.WithContext(ctx) - saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *buffer, _ string, cb func(saveBlobResponse)) { - cb(saveBlobResponse{ - id: restic.Hash(buf.Data), - length: len(buf.Data), - sizeInRepo: len(buf.Data), - known: false, - }) - } - workers := uint(runtime.NumCPU()) pol, err := chunker.RandomPolynomial() if err != nil { t.Fatal(err) } - s := newFileSaver(ctx, wg, saveBlob, pol, workers, workers) + saver := &mockSaver{saved: make(map[string]int)} + s := newFileSaver(ctx, wg, saver, pol, workers) s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) { return meta.ToNode(ignoreXattrListError, t.Logf) } - return s, ctx, wg + return s, saver, ctx, wg } func TestFileSaver(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - files := createTestFiles(t, 15) - startFn := func() {} completeReadingFn := func() {} completeFn := func(*data.Node, ItemStats) {} + files := createTestFiles(t, 15) testFs := fs.Local{} - s, ctx, wg := startFileSaver(ctx, t, testFs) + s, saver, ctx, wg := startFileSaver(ctx, t, testFs) var results []futureNode @@ -89,6 +79,8 @@ func TestFileSaver(t *testing.T) { } } + test.Assert(t, len(saver.saved) == len(files), "expected %d saved files, got %d", len(files), len(saver.saved)) + s.TriggerShutdown() err := wg.Wait() diff --git a/internal/archiver/testing.go b/internal/archiver/testing.go index 666a8c556..6f1195c29 100644 --- a/internal/archiver/testing.go +++ b/internal/archiver/testing.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" ) // TestSnapshot creates a new snapshot of path. @@ -265,19 +266,14 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti t.Helper() tree, err := data.LoadTree(ctx, repo, treeID) - if err != nil { - t.Fatal(err) - return - } + rtest.OK(t, err) var nodeNames []string - for _, node := range tree.Nodes { - nodeNames = append(nodeNames, node.Name) - } - debug.Log("%v (%v) %v", prefix, treeID.Str(), nodeNames) - checked := make(map[string]struct{}) - for _, node := range tree.Nodes { + for item := range tree { + rtest.OK(t, item.Error) + node := item.Node + nodeNames = append(nodeNames, node.Name) nodePrefix := path.Join(prefix, node.Name) entry, ok := dir[node.Name] @@ -316,6 +312,7 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti } } } + debug.Log("%v (%v) %v", prefix, treeID.Str(), nodeNames) for name := range dir { _, ok := checked[name] diff --git a/internal/archiver/tree_saver.go b/internal/archiver/tree_saver.go index d0e802765..8b38b5eb2 100644 --- a/internal/archiver/tree_saver.go +++ b/internal/archiver/tree_saver.go @@ -12,7 +12,7 @@ import ( // treeSaver concurrently saves incoming trees to the repo. type treeSaver struct { - saveBlob saveBlobFn + uploader restic.BlobSaverAsync errFn ErrorFunc ch chan<- saveTreeJob @@ -20,12 +20,12 @@ type treeSaver struct { // newTreeSaver returns a new tree saver. A worker pool with treeWorkers is // started, it is stopped when ctx is cancelled. -func newTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob saveBlobFn, errFn ErrorFunc) *treeSaver { +func newTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, uploader restic.BlobSaverAsync, errFn ErrorFunc) *treeSaver { ch := make(chan saveTreeJob) s := &treeSaver{ ch: ch, - saveBlob: saveBlob, + uploader: uploader, errFn: errFn, } @@ -129,21 +129,35 @@ func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*data.Node, Ite return nil, stats, err } - b := &buffer{Data: buf} - ch := make(chan saveBlobResponse, 1) - s.saveBlob(ctx, restic.TreeBlob, b, job.target, func(res saveBlobResponse) { - ch <- res + var ( + known bool + length int + sizeInRepo int + id restic.ID + ) + + ch := make(chan struct{}, 1) + s.uploader.SaveBlobAsync(ctx, restic.TreeBlob, buf, restic.ID{}, false, func(newID restic.ID, cbKnown bool, cbSizeInRepo int, cbErr error) { + known = cbKnown + length = len(buf) + sizeInRepo = cbSizeInRepo + id = newID + err = cbErr + ch <- struct{}{} }) select { - case sbr := <-ch: - if !sbr.known { + case <-ch: + if err != nil { + return nil, stats, err + } + if !known { stats.TreeBlobs++ - stats.TreeSize += uint64(sbr.length) - stats.TreeSizeInRepo += uint64(sbr.sizeInRepo) + stats.TreeSize += uint64(length) + stats.TreeSizeInRepo += uint64(sizeInRepo) } - node.Subtree = &sbr.id + node.Subtree = &id return node, stats, nil case <-ctx.Done(): return nil, stats, ctx.Err() diff --git a/internal/archiver/tree_saver_test.go b/internal/archiver/tree_saver_test.go index 2a4826444..ed3a148af 100644 --- a/internal/archiver/tree_saver_test.go +++ b/internal/archiver/tree_saver_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "runtime" + "sync" "testing" "github.com/restic/restic/internal/data" @@ -13,13 +14,20 @@ import ( "golang.org/x/sync/errgroup" ) -func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *buffer, _ string, cb func(res saveBlobResponse)) { - cb(saveBlobResponse{ - id: restic.NewRandomID(), - known: false, - length: len(buf.Data), - sizeInRepo: len(buf.Data), - }) +type mockSaver struct { + saved map[string]int + mutex sync.Mutex +} + +func (m *mockSaver) SaveBlobAsync(_ context.Context, _ restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool, cb func(newID restic.ID, known bool, sizeInRepo int, err error)) { + // Fake async operation + go func() { + m.mutex.Lock() + m.saved[string(buf)]++ + m.mutex.Unlock() + + cb(restic.Hash(buf), false, len(buf), nil) + }() } func setupTreeSaver() (context.Context, context.CancelFunc, *treeSaver, func() error) { @@ -30,7 +38,7 @@ func setupTreeSaver() (context.Context, context.CancelFunc, *treeSaver, func() e return err } - b := newTreeSaver(ctx, wg, uint(runtime.NumCPU()), treeSaveHelper, errFn) + b := newTreeSaver(ctx, wg, uint(runtime.NumCPU()), &mockSaver{saved: make(map[string]int)}, errFn) shutdown := func() error { b.TriggerShutdown() diff --git a/internal/backend/local/local_unix.go b/internal/backend/local/local_unix.go index 164785c49..6a6880a2e 100644 --- a/internal/backend/local/local_unix.go +++ b/internal/backend/local/local_unix.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package local diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index fe550a586..ccb1db6ae 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -29,7 +29,7 @@ func TestParse(t *testing.T) { u, err := location.Parse(registry, path) test.OK(t, err) test.Equals(t, "local", u.Scheme) - test.Equals(t, &testConfig{loc: path}, u.Config) + test.Equals(t, any(&testConfig{loc: path}), u.Config) } func TestParseFallback(t *testing.T) { diff --git a/internal/backend/rest/rest_unix_test.go b/internal/backend/rest/rest_unix_test.go index c4f08df0e..2c565f8da 100644 --- a/internal/backend/rest/rest_unix_test.go +++ b/internal/backend/rest/rest_unix_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package rest_test diff --git a/internal/backend/util/defaults_test.go b/internal/backend/util/defaults_test.go index b0efc336f..6cdd058f8 100644 --- a/internal/backend/util/defaults_test.go +++ b/internal/backend/util/defaults_test.go @@ -37,7 +37,7 @@ func TestDefaultLoad(t *testing.T) { return rd, nil }, func(ird io.Reader) error { - rtest.Equals(t, rd, ird) + rtest.Equals(t, io.Reader(rd), ird) return nil }) rtest.OK(t, err) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 28c0f6fa5..0d3a908b8 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -3,7 +3,6 @@ package checker import ( "context" "fmt" - "runtime" "sync" "github.com/restic/restic/internal/data" @@ -12,7 +11,6 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" - "golang.org/x/sync/errgroup" ) // Checker runs various checks on a repository. It is advisable to create an @@ -24,13 +22,17 @@ type Checker struct { *repository.Checker blobRefs struct { sync.Mutex - M restic.BlobSet + M restic.AssociatedBlobSet } trackUnused bool snapshots restic.Lister repo restic.Repository + + // when snapshot filtering is being used + snapshotFilter *data.SnapshotFilter + args []string } type checkerRepository interface { @@ -46,17 +48,24 @@ func New(repo checkerRepository, trackUnused bool) *Checker { trackUnused: trackUnused, } - c.blobRefs.M = restic.NewBlobSet() + c.blobRefs.M = c.repo.NewAssociatedBlobSet() return c } -func (c *Checker) LoadSnapshots(ctx context.Context) error { +func (c *Checker) LoadSnapshots(ctx context.Context, snapshotFilter *data.SnapshotFilter, args []string) error { var err error c.snapshots, err = restic.MemorizeList(ctx, c.repo, restic.SnapshotFile) + c.args = args + c.snapshotFilter = snapshotFilter return err } +// IsFiltered returns true if snapshot filtering is active +func (c *Checker) IsFiltered() bool { + return len(c.args) != 0 || !c.snapshotFilter.Empty() +} + // Error is an error that occurred while checking a repository. type Error struct { TreeID restic.ID @@ -81,31 +90,6 @@ func (e *TreeError) Error() string { return fmt.Sprintf("tree %v: %v", e.ID, e.Errors) } -// checkTreeWorker checks the trees received and sends out errors to errChan. -func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan data.TreeItem, out chan<- error) { - for job := range trees { - debug.Log("check tree %v (tree %v, err %v)", job.ID, job.Tree, job.Error) - - var errs []error - if job.Error != nil { - errs = append(errs, job.Error) - } else { - errs = c.checkTree(job.ID, job.Tree) - } - - if len(errs) == 0 { - continue - } - treeError := &TreeError{ID: job.ID, Errors: errs} - select { - case <-ctx.Done(): - return - case out <- treeError: - debug.Log("tree %v: sent %d errors", treeError.ID, len(treeError.Errors)) - } - } -} - func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.LoaderUnpacked) (ids restic.IDs, errs []error) { err := data.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *data.Snapshot, err error) error { if err != nil { @@ -124,14 +108,43 @@ func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic. return ids, errs } +func (c *Checker) loadActiveTrees(ctx context.Context, snapshotFilter *data.SnapshotFilter, args []string) (trees restic.IDs, errs []error) { + trees = []restic.ID{} + errs = []error{} + + if !c.IsFiltered() { + return loadSnapshotTreeIDs(ctx, c.snapshots, c.repo) + } + + err := snapshotFilter.FindAll(ctx, c.snapshots, c.repo, args, func(_ string, sn *data.Snapshot, err error) error { + if err != nil { + errs = append(errs, err) + return err + } else if sn != nil { + trees = append(trees, *sn.Tree) + } + return nil + }) + + if err != nil { + errs = append(errs, err) + return nil, errs + } + + // track blobs to learn which packs need to be checked + c.trackUnused = true + return trees, errs +} + // Structure checks that for all snapshots all referenced data blobs and // subtrees are available in the index. errChan is closed after all trees have // been traversed. func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan chan<- error) { - trees, errs := loadSnapshotTreeIDs(ctx, c.snapshots, c.repo) + trees, errs := c.loadActiveTrees(ctx, c.snapshotFilter, c.args) p.SetMax(uint64(len(trees))) debug.Log("need to check %d trees from snapshots, %d errs returned", len(trees), len(errs)) + defer close(errChan) for _, err := range errs { select { case <-ctx.Done(): @@ -140,8 +153,7 @@ func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan ch } } - wg, ctx := errgroup.WithContext(ctx) - treeStream := data.StreamTrees(ctx, wg, c.repo, trees, func(treeID restic.ID) bool { + err := data.StreamTrees(ctx, c.repo, trees, p, func(treeID restic.ID) bool { // blobRefs may be accessed in parallel by checkTree c.blobRefs.Lock() h := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob} @@ -150,30 +162,46 @@ func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan ch c.blobRefs.M.Insert(h) c.blobRefs.Unlock() return blobReferenced - }, p) + }, func(treeID restic.ID, err error, nodes data.TreeNodeIterator) error { + debug.Log("check tree %v (err %v)", treeID, err) - defer close(errChan) - // The checkTree worker only processes already decoded trees and is thus CPU-bound - workerCount := runtime.GOMAXPROCS(0) - for i := 0; i < workerCount; i++ { - wg.Go(func() error { - c.checkTreeWorker(ctx, treeStream, errChan) + var errs []error + if err != nil { + errs = append(errs, err) + } else { + errs = c.checkTree(treeID, nodes) + } + if len(errs) == 0 { return nil - }) - } + } - // the wait group should not return an error because no worker returns an + treeError := &TreeError{ID: treeID, Errors: errs} + select { + case <-ctx.Done(): + return nil + case errChan <- treeError: + debug.Log("tree %v: sent %d errors", treeError.ID, len(treeError.Errors)) + } + + return nil + }) + + // StreamTrees should not return an error because no worker returns an // error, so panic if that has changed somehow. - err := wg.Wait() if err != nil { panic(err) } } -func (c *Checker) checkTree(id restic.ID, tree *data.Tree) (errs []error) { +func (c *Checker) checkTree(id restic.ID, tree data.TreeNodeIterator) (errs []error) { debug.Log("checking tree %v", id) - for _, node := range tree.Nodes { + for item := range tree { + if item.Error != nil { + errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("failed to decode tree %v: %w", id, item.Error)}) + break + } + node := item.Node switch node.Type { case data.NodeTypeFile: if node.Content == nil { @@ -245,7 +273,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er c.blobRefs.Lock() defer c.blobRefs.Unlock() - debug.Log("checking %d blobs", len(c.blobRefs.M)) + debug.Log("checking %d blobs", c.blobRefs.M.Len()) ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -259,3 +287,30 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er return blobs, err } + +// ReadPacks wraps repository.ReadPacks: +// in case snapshot filtering is not active it calls repository.ReadPacks() +// with an unmodified parameter list +// Otherwise it calculates the packfiles needed, gets their sizes from the full +// packfile set and submits them to repository.ReadPacks() +func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID]int64) map[restic.ID]int64, p *progress.Counter, errChan chan<- error) { + // no snapshot filtering, pass through + if !c.IsFiltered() { + c.Checker.ReadPacks(ctx, filter, p, errChan) + return + } + + packfileFilter := func(allPacks map[restic.ID]int64) map[restic.ID]int64 { + filteredPacks := make(map[restic.ID]int64) + // convert used blobs into their encompassing packfiles + for bh := range c.blobRefs.M.Keys() { + for _, pb := range c.repo.LookupBlob(bh.Type, bh.ID) { + filteredPacks[pb.PackID] = allPacks[pb.PackID] + } + } + + return filter(filteredPacks) + } + + c.Checker.ReadPacks(ctx, packfileFilter, p, errChan) +} diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 960942d80..106ffd6b3 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -46,7 +46,7 @@ func checkPacks(chkr *checker.Checker) []error { } func checkStruct(chkr *checker.Checker) []error { - err := chkr.LoadSnapshots(context.TODO()) + err := chkr.LoadSnapshots(context.TODO(), &data.SnapshotFilter{}, nil) if err != nil { return []error{err} } @@ -522,21 +522,18 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { Size: 42, Content: restic.IDs{restic.TestParseID("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")}, } - damagedTree := &data.Tree{ - Nodes: []*data.Node{damagedNode}, - } + damagedNodes := []*data.Node{damagedNode} var id restic.ID - test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { - var err error - id, err = data.SaveTree(ctx, uploader, damagedTree) - return err + test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + id = data.TestSaveNodes(t, ctx, uploader, damagedNodes) + return nil })) buf, err := repo.LoadBlob(ctx, restic.TreeBlob, id, nil) test.OK(t, err) - test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { var err error _, _, _, err = uploader.SaveBlob(ctx, restic.DataBlob, buf, id, false) return err @@ -556,15 +553,12 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { Subtree: &id, } - rootTree := &data.Tree{ - Nodes: []*data.Node{malNode, dirNode}, - } + rootNodes := []*data.Node{malNode, dirNode} var rootID restic.ID - test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { - var err error - rootID, err = data.SaveTree(ctx, uploader, rootTree) - return err + test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + rootID = data.TestSaveNodes(t, ctx, uploader, rootNodes) + return nil })) snapshot, err := data.NewSnapshot([]string{"/damaged"}, []string{"test"}, "foo", time.Now()) diff --git a/internal/checker/testing.go b/internal/checker/testing.go index 1358bf362..eaa16382a 100644 --- a/internal/checker/testing.go +++ b/internal/checker/testing.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" ) @@ -20,7 +21,7 @@ func TestCheckRepo(t testing.TB, repo checkerRepository) { t.Fatalf("errors loading index: %v", hints) } - err := chkr.LoadSnapshots(context.TODO()) + err := chkr.LoadSnapshots(context.TODO(), &data.SnapshotFilter{}, nil) if err != nil { t.Error(err) } diff --git a/internal/data/find.go b/internal/data/find.go index 14d64670e..5fadcba93 100644 --- a/internal/data/find.go +++ b/internal/data/find.go @@ -6,7 +6,6 @@ import ( "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" - "golang.org/x/sync/errgroup" ) // FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data @@ -14,8 +13,7 @@ import ( func FindUsedBlobs(ctx context.Context, repo restic.Loader, treeIDs restic.IDs, blobs restic.FindBlobSet, p *progress.Counter) error { var lock sync.Mutex - wg, ctx := errgroup.WithContext(ctx) - treeStream := StreamTrees(ctx, wg, repo, treeIDs, func(treeID restic.ID) bool { + return StreamTrees(ctx, repo, treeIDs, p, func(treeID restic.ID) bool { // locking is necessary the goroutine below concurrently adds data blobs lock.Lock() h := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob} @@ -24,26 +22,24 @@ func FindUsedBlobs(ctx context.Context, repo restic.Loader, treeIDs restic.IDs, blobs.Insert(h) lock.Unlock() return blobReferenced - }, p) + }, func(_ restic.ID, err error, nodes TreeNodeIterator) error { + if err != nil { + return err + } - wg.Go(func() error { - for tree := range treeStream { - if tree.Error != nil { - return tree.Error + for item := range nodes { + if item.Error != nil { + return item.Error } - lock.Lock() - for _, node := range tree.Nodes { - switch node.Type { - case NodeTypeFile: - for _, blob := range node.Content { - blobs.Insert(restic.BlobHandle{ID: blob, Type: restic.DataBlob}) - } + switch item.Node.Type { + case NodeTypeFile: + for _, blob := range item.Node.Content { + blobs.Insert(restic.BlobHandle{ID: blob, Type: restic.DataBlob}) } } lock.Unlock() } return nil }) - return wg.Wait() } diff --git a/internal/data/testdata/used_blobs_snapshot0 b/internal/data/testdata/used_blobs_snapshot0 index cc789f043..85ac807f0 100644 --- a/internal/data/testdata/used_blobs_snapshot0 +++ b/internal/data/testdata/used_blobs_snapshot0 @@ -1,9 +1,9 @@ {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"} {"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"} -{"ID":"08a650e4d7575177ddeabf6a96896b76fa7e621aa3dd75e77293f22ce6c0c420","Type":"tree"} {"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"} -{"ID":"435b9207cd489b41a7d119e0d75eab2a861e2b3c8d4d12ac51873ff76be0cf73","Type":"tree"} +{"ID":"3cb26562e6849003adffc5e1dcf9a58a9d151ea4203bd451f6359d7cc0328104","Type":"tree"} {"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"} +{"ID":"4bb52083c8a467921e8ed4139f7be3e282bad8d25d0056145eadd3962aed0127","Type":"tree"} {"ID":"4e352975938a29711c3003c498185972235af261a6cf8cf700a8a6ee4f914b05","Type":"data"} {"ID":"606772eacb7fe1a79267088dcadd13431914854faf1d39d47fe99a26b9fecdcb","Type":"data"} {"ID":"6b5fd3a9baf615489c82a99a71f9917bf9a2d82d5f640d7f47d175412c4b8d19","Type":"data"} @@ -14,10 +14,10 @@ {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"} {"ID":"b11f4dd9d2722b3325186f57cd13a71a3af7791118477f355b49d101104e4c22","Type":"data"} {"ID":"b1f2ae9d748035e5bd9a87f2579405166d150c6560d8919496f02855e1c36cf9","Type":"data"} +{"ID":"b326b56e1b4c5c3b80e449fc40abcada21b5bd7ff12ce02236a2d289b89dcea7","Type":"tree"} {"ID":"b5ba06039224566a09555abd089de7a693660154991295122fa72b0a3adc4150","Type":"data"} {"ID":"b7040572b44cbfea8b784ecf8679c3d75cefc1cd3d12ed783ca0d8e5d124a60f","Type":"data"} {"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"} {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"} {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"} {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"} -{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"} diff --git a/internal/data/testdata/used_blobs_snapshot1 b/internal/data/testdata/used_blobs_snapshot1 index aa840294a..e7d66c7ef 100644 --- a/internal/data/testdata/used_blobs_snapshot1 +++ b/internal/data/testdata/used_blobs_snapshot1 @@ -1,6 +1,8 @@ {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"} {"ID":"18dcaa1a676823c909aafabbb909652591915eebdde4f9a65cee955157583494","Type":"data"} +{"ID":"428b3f50bcfdcb9a85b87f9401d8947b2c8a2f807c19c00491626e3ee890075e","Type":"tree"} {"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"} +{"ID":"55416727dd211e5f208b70fc9a3d60d34484626279717be87843a7535f997404","Type":"tree"} {"ID":"6824d08e63a598c02b364e25f195e64758494b5944f06c921ff30029e1e4e4bf","Type":"data"} {"ID":"72b6eb0fd0d87e00392f8b91efc1a4c3f7f5c0c76f861b38aea054bc9d43463b","Type":"data"} {"ID":"8192279e4b56e1644dcff715d5e08d875cd5713349139d36d142ed28364d8e00","Type":"data"} @@ -10,6 +12,4 @@ {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"} {"ID":"cc4cab5b20a3a88995f8cdb8b0698d67a32dbc5b54487f03cb612c30a626af39","Type":"data"} {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"} -{"ID":"e9f3c4fe78e903cba60d310a9668c42232c8274b3f29b5ecebb6ff1aaeabd7e3","Type":"tree"} {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"} -{"ID":"ff58f76c2313e68aa9aaaece855183855ac4ff682910404c2ae33dc999ebaca2","Type":"tree"} diff --git a/internal/data/testdata/used_blobs_snapshot2 b/internal/data/testdata/used_blobs_snapshot2 index 3ed193f53..029bcbc6e 100644 --- a/internal/data/testdata/used_blobs_snapshot2 +++ b/internal/data/testdata/used_blobs_snapshot2 @@ -1,24 +1,24 @@ {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"} {"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"} {"ID":"0b88f99abc5ac71c54b3e8263c52ecb7d8903462779afdb3c8176ec5c4bb04fb","Type":"data"} -{"ID":"0e1a817fca83f569d1733b11eba14b6c9b176e41bca3644eed8b29cb907d84d3","Type":"tree"} {"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"} {"ID":"27917462f89cecae77a4c8fb65a094b9b75a917f13794c628b1640b17f4c4981","Type":"data"} +{"ID":"2b8ebd79732fcfec50ec94429cfd404d531c93defed78832a597d0fe8de64b96","Type":"tree"} {"ID":"32745e4b26a5883ecec272c9fbfe7f3c9835c9ab41c9a2baa4d06f319697a0bd","Type":"data"} {"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"} {"ID":"4e352975938a29711c3003c498185972235af261a6cf8cf700a8a6ee4f914b05","Type":"data"} {"ID":"6824d08e63a598c02b364e25f195e64758494b5944f06c921ff30029e1e4e4bf","Type":"data"} {"ID":"6b5fd3a9baf615489c82a99a71f9917bf9a2d82d5f640d7f47d175412c4b8d19","Type":"data"} +{"ID":"721d803612a2565f9be9581048c5d899c14a65129dabafbb0e43bba89684a63a","Type":"tree"} +{"ID":"9103a50221ecf6684c1e1652adacb4a85afb322d81a74ecd7477930ecf4774fc","Type":"tree"} {"ID":"95c97192efa810ccb1cee112238dca28673fbffce205d75ce8cc990a31005a51","Type":"data"} {"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"} {"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"} {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"} +{"ID":"ac08ce34ba4f8123618661bef2425f7028ffb9ac740578a3ee88684d2523fee8","Type":"tree"} +{"ID":"b326b56e1b4c5c3b80e449fc40abcada21b5bd7ff12ce02236a2d289b89dcea7","Type":"tree"} {"ID":"b6a7e8d2aa717e0a6bd68abab512c6b566074b5a6ca2edf4cd446edc5857d732","Type":"data"} -{"ID":"bad84ed273c5fbfb40aa839a171675b7f16f5e67f3eaf4448730caa0ee27297c","Type":"tree"} -{"ID":"bfc2fdb527b0c9f66bbb8d4ff1c44023cc2414efcc7f0831c10debab06bb4388","Type":"tree"} {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"} -{"ID":"d1d3137eb08de6d8c5d9f44788c45a9fea9bb082e173bed29a0945b3347f2661","Type":"tree"} {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"} {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"} {"ID":"f3cd67d9c14d2a81663d63522ab914e465b021a3b65e2f1ea6caf7478f2ec139","Type":"data"} -{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"} diff --git a/internal/data/testing.go b/internal/data/testing.go index be4ab4edb..19f7fa7b3 100644 --- a/internal/data/testing.go +++ b/internal/data/testing.go @@ -5,12 +5,14 @@ import ( "fmt" "io" "math/rand" + "slices" + "strings" "testing" "time" "github.com/restic/chunker" "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" ) // fakeFile returns a reader which yields deterministic pseudo-random data. @@ -72,22 +74,21 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, uploader restic.BlobSave rnd := rand.NewSource(seed) numNodes := int(rnd.Int63() % maxNodes) - var tree Tree + var nodes []*Node for i := 0; i < numNodes; i++ { - // randomly select the type of the node, either tree (p = 1/4) or file (p = 3/4). if depth > 1 && rnd.Int63()%4 == 0 { treeSeed := rnd.Int63() % maxSeed id := fs.saveTree(ctx, uploader, treeSeed, depth-1) node := &Node{ - Name: fmt.Sprintf("dir-%v", treeSeed), + Name: fmt.Sprintf("dir-%v", i), Type: NodeTypeDir, Mode: 0755, Subtree: &id, } - tree.Nodes = append(tree.Nodes, node) + nodes = append(nodes, node) continue } @@ -95,22 +96,31 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, uploader restic.BlobSave fileSize := (maxFileSize / maxSeed) * fileSeed node := &Node{ - Name: fmt.Sprintf("file-%v", fileSeed), + Name: fmt.Sprintf("file-%v", i), Type: NodeTypeFile, Mode: 0644, Size: uint64(fileSize), } node.Content = fs.saveFile(ctx, uploader, fakeFile(fileSeed, fileSize)) - tree.Nodes = append(tree.Nodes, node) + nodes = append(nodes, node) } - tree.Sort() + return TestSaveNodes(fs.t, ctx, uploader, nodes) +} - id, err := SaveTree(ctx, uploader, &tree) - if err != nil { - fs.t.Fatalf("SaveTree returned error: %v", err) +//nolint:revive // as this is a test helper, t should go first +func TestSaveNodes(t testing.TB, ctx context.Context, uploader restic.BlobSaver, nodes []*Node) restic.ID { + slices.SortFunc(nodes, func(a, b *Node) int { + return strings.Compare(a.Name, b.Name) + }) + treeWriter := NewTreeWriter(uploader) + for _, node := range nodes { + err := treeWriter.AddNode(node) + rtest.OK(t, err) } + id, err := treeWriter.Finalize(ctx) + rtest.OK(t, err) return id } @@ -136,7 +146,7 @@ func TestCreateSnapshot(t testing.TB, repo restic.Repository, at time.Time, dept } var treeID restic.ID - test.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { treeID = fs.saveTree(ctx, uploader, seed, depth) return nil })) @@ -188,3 +198,49 @@ func TestLoadAllSnapshots(ctx context.Context, repo restic.ListerLoaderUnpacked, return snapshots, nil } + +// TestTreeMap returns the trees from the map on LoadTree. +type TestTreeMap map[restic.ID][]byte + +func (t TestTreeMap) LoadBlob(_ context.Context, tpe restic.BlobType, id restic.ID, _ []byte) ([]byte, error) { + if tpe != restic.TreeBlob { + return nil, fmt.Errorf("can only load trees") + } + tree, ok := t[id] + if !ok { + return nil, fmt.Errorf("tree not found") + } + return tree, nil +} + +func (t TestTreeMap) Connections() uint { + return 2 +} + +// TestWritableTreeMap also support saving +type TestWritableTreeMap struct { + TestTreeMap +} + +func (t TestWritableTreeMap) SaveBlob(_ context.Context, tpe restic.BlobType, buf []byte, id restic.ID, _ bool) (newID restic.ID, known bool, size int, err error) { + if tpe != restic.TreeBlob { + return restic.ID{}, false, 0, fmt.Errorf("can only save trees") + } + + if id.IsNull() { + id = restic.Hash(buf) + } + _, ok := t.TestTreeMap[id] + if ok { + return id, false, 0, nil + } + + t.TestTreeMap[id] = append([]byte{}, buf...) + return id, true, len(buf), nil +} + +func (t TestWritableTreeMap) Dump(test testing.TB) { + for k, v := range t.TestTreeMap { + test.Logf("%v: %v", k, string(v)) + } +} diff --git a/internal/data/tree.go b/internal/data/tree.go index 7eff126d4..4673c6a92 100644 --- a/internal/data/tree.go +++ b/internal/data/tree.go @@ -5,146 +5,247 @@ import ( "context" "encoding/json" "fmt" + + "io" + "iter" "path" - "sort" "strings" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - - "github.com/restic/restic/internal/debug" ) -// Tree is an ordered list of nodes. -type Tree struct { - Nodes []*Node `json:"nodes"` -} - -// NewTree creates a new tree object with the given initial capacity. -func NewTree(capacity int) *Tree { - return &Tree{ - Nodes: make([]*Node, 0, capacity), - } -} - -func (t *Tree) String() string { - return fmt.Sprintf("Tree<%d nodes>", len(t.Nodes)) -} - -// Equals returns true if t and other have exactly the same nodes. -func (t *Tree) Equals(other *Tree) bool { - if len(t.Nodes) != len(other.Nodes) { - debug.Log("tree.Equals(): trees have different number of nodes") - return false - } - - for i := 0; i < len(t.Nodes); i++ { - if !t.Nodes[i].Equals(*other.Nodes[i]) { - debug.Log("tree.Equals(): node %d is different:", i) - debug.Log(" %#v", t.Nodes[i]) - debug.Log(" %#v", other.Nodes[i]) - return false - } - } - - return true -} - -// Insert adds a new node at the correct place in the tree. -func (t *Tree) Insert(node *Node) error { - pos, found := t.find(node.Name) - if found != nil { - return errors.Errorf("node %q already present", node.Name) - } - - // https://github.com/golang/go/wiki/SliceTricks - t.Nodes = append(t.Nodes, nil) - copy(t.Nodes[pos+1:], t.Nodes[pos:]) - t.Nodes[pos] = node - - return nil -} - -func (t *Tree) find(name string) (int, *Node) { - pos := sort.Search(len(t.Nodes), func(i int) bool { - return t.Nodes[i].Name >= name - }) - - if pos < len(t.Nodes) && t.Nodes[pos].Name == name { - return pos, t.Nodes[pos] - } - - return pos, nil -} - -// Find returns a node with the given name, or nil if none could be found. -func (t *Tree) Find(name string) *Node { - if t == nil { - return nil - } - - _, node := t.find(name) - return node -} - -// Sort sorts the nodes by name. -func (t *Tree) Sort() { - list := Nodes(t.Nodes) - sort.Sort(list) - t.Nodes = list -} - -// Subtrees returns a slice of all subtree IDs of the tree. -func (t *Tree) Subtrees() (trees restic.IDs) { - for _, node := range t.Nodes { - if node.Type == NodeTypeDir && node.Subtree != nil { - trees = append(trees, *node.Subtree) - } - } - - return trees -} - -// LoadTree loads a tree from the repository. -func LoadTree(ctx context.Context, r restic.BlobLoader, id restic.ID) (*Tree, error) { - debug.Log("load tree %v", id) - - buf, err := r.LoadBlob(ctx, restic.TreeBlob, id, nil) - if err != nil { - return nil, err - } - - t := &Tree{} - err = json.Unmarshal(buf, t) - if err != nil { - return nil, err - } - - return t, nil -} - -// SaveTree stores a tree into the repository and returns the ID. The ID is -// checked against the index. The tree is only stored when the index does not -// contain the ID. -func SaveTree(ctx context.Context, r restic.BlobSaver, t *Tree) (restic.ID, error) { - buf, err := json.Marshal(t) - if err != nil { - return restic.ID{}, errors.Wrap(err, "MarshalJSON") - } - - // append a newline so that the data is always consistent (json.Encoder - // adds a newline after each object) - buf = append(buf, '\n') - - id, _, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) - return id, err -} +// For documentation purposes only: +// // Tree is an ordered list of nodes. +// type Tree struct { +// Nodes []*Node `json:"nodes"` +// } var ErrTreeNotOrdered = errors.New("nodes are not ordered or duplicate") +type treeIterator struct { + dec json.Decoder + started bool +} + +type NodeOrError struct { + Node *Node + Error error +} + +type TreeNodeIterator = iter.Seq[NodeOrError] + +func NewTreeNodeIterator(rd io.Reader) (TreeNodeIterator, error) { + t := &treeIterator{ + dec: *json.NewDecoder(rd), + } + + err := t.init() + if err != nil { + return nil, err + } + + return func(yield func(NodeOrError) bool) { + if t.started { + panic("tree iterator is single use only") + } + t.started = true + for { + n, err := t.next() + if err != nil && errors.Is(err, io.EOF) { + return + } + if !yield(NodeOrError{Node: n, Error: err}) { + return + } + // errors are final + if err != nil { + return + } + } + }, nil +} + +func (t *treeIterator) init() error { + // A tree is expected to be encoded as a JSON object with a single key "nodes". + // However, for future-proofness, we allow unknown keys before and after the "nodes" key. + // The following is the expected format: + // `{"nodes":[...]}` + + if err := t.assertToken(json.Delim('{')); err != nil { + return err + } + // Skip unknown keys until we find "nodes" + for { + token, err := t.dec.Token() + if err != nil { + return err + } + key, ok := token.(string) + if !ok { + return errors.Errorf("error decoding tree: expected string key, got %v", token) + } + if key == "nodes" { + // Found "nodes", proceed to read the array + if err := t.assertToken(json.Delim('[')); err != nil { + return err + } + return nil + } + // Unknown key, decode its value into RawMessage and discard it + var raw json.RawMessage + if err := t.dec.Decode(&raw); err != nil { + return err + } + } +} + +func (t *treeIterator) next() (*Node, error) { + if t.dec.More() { + var n Node + err := t.dec.Decode(&n) + if err != nil { + return nil, err + } + return &n, nil + } + + if err := t.assertToken(json.Delim(']')); err != nil { + return nil, err + } + // Skip unknown keys after the array until we find the closing brace + for { + token, err := t.dec.Token() + if err != nil { + return nil, err + } + if token == json.Delim('}') { + return nil, io.EOF + } + // We have an unknown key, decode its value into RawMessage and discard it + var raw json.RawMessage + if err := t.dec.Decode(&raw); err != nil { + return nil, err + } + } +} + +func (t *treeIterator) assertToken(token json.Token) error { + to, err := t.dec.Token() + if err != nil { + return err + } + if to != token { + return errors.Errorf("error decoding tree: expected %v, got %v", token, to) + } + return nil +} + +func LoadTree(ctx context.Context, loader restic.BlobLoader, content restic.ID) (TreeNodeIterator, error) { + rd, err := loader.LoadBlob(ctx, restic.TreeBlob, content, nil) + if err != nil { + return nil, err + } + return NewTreeNodeIterator(bytes.NewReader(rd)) +} + +type TreeFinder struct { + next func() (NodeOrError, bool) + stop func() + current *Node + last string +} + +func NewTreeFinder(tree TreeNodeIterator) *TreeFinder { + if tree == nil { + return &TreeFinder{stop: func() {}} + } + next, stop := iter.Pull(tree) + return &TreeFinder{next: next, stop: stop} +} + +// Find finds the node with the given name. If the node is not found, it returns nil. +// If Find was called before, the new name must be strictly greater than the last name. +func (t *TreeFinder) Find(name string) (*Node, error) { + if t.next == nil { + return nil, nil + } + if name <= t.last { + return nil, errors.Errorf("name %q is not greater than last name %q", name, t.last) + } + t.last = name + // loop until `t.current.Name` is >= name + for t.current == nil || t.current.Name < name { + current, ok := t.next() + if current.Error != nil { + return nil, current.Error + } + if !ok { + return nil, nil + } + t.current = current.Node + } + + if t.current.Name == name { + // forget the current node to free memory as early as possible + current := t.current + t.current = nil + return current, nil + } + // we have already passed the name + return nil, nil +} + +func (t *TreeFinder) Close() { + t.stop() +} + +type TreeWriter struct { + builder *TreeJSONBuilder + saver restic.BlobSaver +} + +func NewTreeWriter(saver restic.BlobSaver) *TreeWriter { + builder := NewTreeJSONBuilder() + return &TreeWriter{builder: builder, saver: saver} +} + +func (t *TreeWriter) AddNode(node *Node) error { + return t.builder.AddNode(node) +} + +func (t *TreeWriter) Finalize(ctx context.Context) (restic.ID, error) { + buf, err := t.builder.Finalize() + if err != nil { + return restic.ID{}, err + } + id, _, _, err := t.saver.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) + return id, err +} + +// Count returns the number of nodes in the tree +func (t *TreeWriter) Count() int { + return t.builder.Count() +} + +func SaveTree(ctx context.Context, saver restic.BlobSaver, nodes TreeNodeIterator) (restic.ID, error) { + treeWriter := NewTreeWriter(saver) + for item := range nodes { + if item.Error != nil { + return restic.ID{}, item.Error + } + err := treeWriter.AddNode(item.Node) + if err != nil { + return restic.ID{}, err + } + } + return treeWriter.Finalize(ctx) +} + type TreeJSONBuilder struct { - buf bytes.Buffer - lastName string + buf bytes.Buffer + lastName string + countNodes int } func NewTreeJSONBuilder() *TreeJSONBuilder { @@ -167,6 +268,7 @@ func (builder *TreeJSONBuilder) AddNode(node *Node) error { return err } _, _ = builder.buf.Write(val) + builder.countNodes++ return nil } @@ -180,6 +282,11 @@ func (builder *TreeJSONBuilder) Finalize() ([]byte, error) { return buf, nil } +// Count returns the number of nodes in the tree +func (builder *TreeJSONBuilder) Count() int { + return builder.countNodes +} + func FindTreeDirectory(ctx context.Context, repo restic.BlobLoader, id *restic.ID, dir string) (*restic.ID, error) { if id == nil { return nil, errors.New("tree id is null") @@ -197,7 +304,12 @@ func FindTreeDirectory(ctx context.Context, repo restic.BlobLoader, id *restic.I if err != nil { return nil, fmt.Errorf("path %s: %w", subfolder, err) } - node := tree.Find(name) + finder := NewTreeFinder(tree) + node, err := finder.Find(name) + finder.Close() + if err != nil { + return nil, fmt.Errorf("path %s: %w", subfolder, err) + } if node == nil { return nil, fmt.Errorf("path %s: not found", subfolder) } @@ -218,5 +330,109 @@ func FindTreeNode(ctx context.Context, repo restic.BlobLoader, id *restic.ID, no if err != nil { return nil, err } - return tree.Find(path.Base(nodepath)), nil + finder := NewTreeFinder(tree) + defer finder.Close() + return finder.Find(path.Base(nodepath)) +} + +type peekableNodeIterator struct { + iter func() (NodeOrError, bool) + stop func() + value *Node +} + +func newPeekableNodeIterator(tree TreeNodeIterator) (*peekableNodeIterator, error) { + iter, stop := iter.Pull(tree) + it := &peekableNodeIterator{iter: iter, stop: stop} + err := it.Next() + if err != nil { + it.Close() + return nil, err + } + return it, nil +} + +func (i *peekableNodeIterator) Next() error { + item, ok := i.iter() + if item.Error != nil || !ok { + i.value = nil + return item.Error + } + i.value = item.Node + return nil +} + +func (i *peekableNodeIterator) Peek() *Node { + return i.value +} + +func (i *peekableNodeIterator) Close() { + i.stop() +} + +type DualTree struct { + Tree1 *Node + Tree2 *Node + Error error +} + +// DualTreeIterator iterates over two trees in parallel. It returns a sequence of DualTree structs. +// The sequence is terminated when both trees are exhausted. The error field must be checked before +// accessing any of the nodes. +func DualTreeIterator(tree1, tree2 TreeNodeIterator) iter.Seq[DualTree] { + started := false + return func(yield func(DualTree) bool) { + if started { + panic("tree iterator is single use only") + } + started = true + iter1, err := newPeekableNodeIterator(tree1) + if err != nil { + yield(DualTree{Tree1: nil, Tree2: nil, Error: err}) + return + } + defer iter1.Close() + iter2, err := newPeekableNodeIterator(tree2) + if err != nil { + yield(DualTree{Tree1: nil, Tree2: nil, Error: err}) + return + } + defer iter2.Close() + + for { + node1 := iter1.Peek() + node2 := iter2.Peek() + if node1 == nil && node2 == nil { + // both iterators are exhausted + break + } else if node1 != nil && node2 != nil { + // if both nodes have a different name, only keep the first one + if node1.Name < node2.Name { + node2 = nil + } else if node1.Name > node2.Name { + node1 = nil + } + } + + // non-nil nodes will be processed in the following, so advance the corresponding iterator + if node1 != nil { + if err = iter1.Next(); err != nil { + break + } + } + if node2 != nil { + if err = iter2.Next(); err != nil { + break + } + } + + if !yield(DualTree{Tree1: node1, Tree2: node2, Error: err}) { + return + } + } + if err != nil { + yield(DualTree{Tree1: nil, Tree2: nil, Error: err}) + return + } + } } diff --git a/internal/data/tree_stream.go b/internal/data/tree_stream.go index c7d3588b5..1f832a731 100644 --- a/internal/data/tree_stream.go +++ b/internal/data/tree_stream.go @@ -2,26 +2,20 @@ package data import ( "context" - "errors" "runtime" "sync" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) -// TreeItem is used to return either an error or the tree for a tree id -type TreeItem struct { - restic.ID - Error error - *Tree -} - type trackedTreeItem struct { - TreeItem - rootIdx int + restic.ID + Subtrees restic.IDs + rootIdx int } type trackedID struct { @@ -29,35 +23,87 @@ type trackedID struct { rootIdx int } +// subtreesCollector wraps a TreeNodeIterator and returns a new iterator that collects the subtrees. +func subtreesCollector(tree TreeNodeIterator) (TreeNodeIterator, func() restic.IDs) { + subtrees := restic.IDs{} + isComplete := false + + return func(yield func(NodeOrError) bool) { + for item := range tree { + if !yield(item) { + return + } + // be defensive and check for nil subtree as this code is also used by the checker + if item.Node != nil && item.Node.Type == NodeTypeDir && item.Node.Subtree != nil { + subtrees = append(subtrees, *item.Node.Subtree) + } + } + isComplete = true + }, func() restic.IDs { + if !isComplete { + panic("tree was not read completely") + } + return subtrees + } +} + // loadTreeWorker loads trees from repo and sends them to out. -func loadTreeWorker(ctx context.Context, repo restic.Loader, - in <-chan trackedID, out chan<- trackedTreeItem) { +func loadTreeWorker( + ctx context.Context, + repo restic.Loader, + in <-chan trackedID, + process func(id restic.ID, error error, nodes TreeNodeIterator) error, + out chan<- trackedTreeItem, +) error { for treeID := range in { tree, err := LoadTree(ctx, repo, treeID.ID) + if tree == nil && err == nil { + err = errors.New("tree is nil and error is nil") + } debug.Log("load tree %v (%v) returned err: %v", tree, treeID, err) - job := trackedTreeItem{TreeItem: TreeItem{ID: treeID.ID, Error: err, Tree: tree}, rootIdx: treeID.rootIdx} + + // wrap iterator to collect subtrees while `process` iterates over `tree` + var collectSubtrees func() restic.IDs + if tree != nil { + tree, collectSubtrees = subtreesCollector(tree) + } + + err = process(treeID.ID, err, tree) + if err != nil { + return err + } + + // assume that the number of subtrees is within reasonable limits, such that the memory usage is not a problem + var subtrees restic.IDs + if collectSubtrees != nil { + subtrees = collectSubtrees() + } + + job := trackedTreeItem{ID: treeID.ID, Subtrees: subtrees, rootIdx: treeID.rootIdx} select { case <-ctx.Done(): - return + return nil case out <- job: } } + return nil } +// filterTree receives the result of a tree load and queues new trees for loading and processing. func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, loaderChan chan<- trackedID, hugeTreeLoaderChan chan<- trackedID, - in <-chan trackedTreeItem, out chan<- TreeItem, skip func(tree restic.ID) bool, p *progress.Counter) { + in <-chan trackedTreeItem, skip func(tree restic.ID) bool, p *progress.Counter) { var ( inCh = in - outCh chan<- TreeItem loadCh chan<- trackedID - job TreeItem nextTreeID trackedID outstandingLoadTreeJobs = 0 ) + // tracks how many trees are currently waiting to be processed for a given root tree rootCounter := make([]int, len(trees)) + // build initial backlog backlog := make([]trackedID, 0, len(trees)) for idx, id := range trees { backlog = append(backlog, trackedID{ID: id, rootIdx: idx}) @@ -65,6 +111,7 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load } for { + // if no tree is waiting to be sent, pick the next one if loadCh == nil && len(backlog) > 0 { // process last added ids first, that is traverse the tree in depth-first order ln := len(backlog) - 1 @@ -86,7 +133,8 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load } } - if loadCh == nil && outCh == nil && outstandingLoadTreeJobs == 0 { + // loadCh is only nil at this point if the backlog is empty + if loadCh == nil && outstandingLoadTreeJobs == 0 { debug.Log("backlog is empty, all channels nil, exiting") return } @@ -103,7 +151,6 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load if !ok { debug.Log("input channel closed") inCh = nil - in = nil continue } @@ -111,58 +158,47 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load rootCounter[j.rootIdx]-- debug.Log("input job tree %v", j.ID) - - if j.Error != nil { - debug.Log("received job with error: %v (tree %v, ID %v)", j.Error, j.Tree, j.ID) - } else if j.Tree == nil { - debug.Log("received job with nil tree pointer: %v (ID %v)", j.Error, j.ID) - // send a new job with the new error instead of the old one - j = trackedTreeItem{TreeItem: TreeItem{ID: j.ID, Error: errors.New("tree is nil and error is nil")}, rootIdx: j.rootIdx} - } else { - subtrees := j.Tree.Subtrees() - debug.Log("subtrees for tree %v: %v", j.ID, subtrees) - // iterate backwards over subtree to compensate backwards traversal order of nextTreeID selection - for i := len(subtrees) - 1; i >= 0; i-- { - id := subtrees[i] - if id.IsNull() { - // We do not need to raise this error here, it is - // checked when the tree is checked. Just make sure - // that we do not add any null IDs to the backlog. - debug.Log("tree %v has nil subtree", j.ID) - continue - } - backlog = append(backlog, trackedID{ID: id, rootIdx: j.rootIdx}) - rootCounter[j.rootIdx]++ + // iterate backwards over subtree to compensate backwards traversal order of nextTreeID selection + for i := len(j.Subtrees) - 1; i >= 0; i-- { + id := j.Subtrees[i] + if id.IsNull() { + // We do not need to raise this error here, it is + // checked when the tree is checked. Just make sure + // that we do not add any null IDs to the backlog. + debug.Log("tree %v has nil subtree", j.ID) + continue } + backlog = append(backlog, trackedID{ID: id, rootIdx: j.rootIdx}) + rootCounter[j.rootIdx]++ } + // the progress check must happen after j.Subtrees was added to the backlog if p != nil && rootCounter[j.rootIdx] == 0 { p.Add(1) } - - job = j.TreeItem - outCh = out - inCh = nil - - case outCh <- job: - debug.Log("tree sent to process: %v", job.ID) - outCh = nil - inCh = in } } } // StreamTrees iteratively loads the given trees and their subtrees. The skip method -// is guaranteed to always be called from the same goroutine. To shutdown the started -// goroutines, either read all items from the channel or cancel the context. Then `Wait()` -// on the errgroup until all goroutines were stopped. -func StreamTrees(ctx context.Context, wg *errgroup.Group, repo restic.Loader, trees restic.IDs, skip func(tree restic.ID) bool, p *progress.Counter) <-chan TreeItem { +// is guaranteed to always be called from the same goroutine. The process function is +// directly called from the worker goroutines. It MUST read `nodes` until it returns an +// error or completes. If the process function returns an error, then StreamTrees will +// abort and return the error. +func StreamTrees( + ctx context.Context, + repo restic.Loader, + trees restic.IDs, + p *progress.Counter, + skip func(tree restic.ID) bool, + process func(id restic.ID, error error, nodes TreeNodeIterator) error, +) error { loaderChan := make(chan trackedID) hugeTreeChan := make(chan trackedID, 10) loadedTreeChan := make(chan trackedTreeItem) - treeStream := make(chan TreeItem) var loadTreeWg sync.WaitGroup + wg, ctx := errgroup.WithContext(ctx) // decoding a tree can take quite some time such that this can be both CPU- or IO-bound // one extra worker to handle huge tree blobs workerCount := int(repo.Connections()) + runtime.GOMAXPROCS(0) + 1 @@ -174,8 +210,7 @@ func StreamTrees(ctx context.Context, wg *errgroup.Group, repo restic.Loader, tr loadTreeWg.Add(1) wg.Go(func() error { defer loadTreeWg.Done() - loadTreeWorker(ctx, repo, workerLoaderChan, loadedTreeChan) - return nil + return loadTreeWorker(ctx, repo, workerLoaderChan, process, loadedTreeChan) }) } @@ -189,9 +224,8 @@ func StreamTrees(ctx context.Context, wg *errgroup.Group, repo restic.Loader, tr wg.Go(func() error { defer close(loaderChan) defer close(hugeTreeChan) - defer close(treeStream) - filterTrees(ctx, repo, trees, loaderChan, hugeTreeChan, loadedTreeChan, treeStream, skip, p) + filterTrees(ctx, repo, trees, loaderChan, hugeTreeChan, loadedTreeChan, skip, p) return nil }) - return treeStream + return wg.Wait() } diff --git a/internal/data/tree_test.go b/internal/data/tree_test.go index 9164f4da1..47fc4b9a0 100644 --- a/internal/data/tree_test.go +++ b/internal/data/tree_test.go @@ -4,9 +4,12 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "path/filepath" + "slices" "strconv" + "strings" "testing" "github.com/restic/restic/internal/archiver" @@ -105,37 +108,45 @@ func TestNodeComparison(t *testing.T) { func TestEmptyLoadTree(t *testing.T) { repo := repository.TestRepository(t) - tree := data.NewTree(0) + nodes := []*data.Node{} var id restic.ID - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { - var err error + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { // save tree - id, err = data.SaveTree(ctx, uploader, tree) - return err + id = data.TestSaveNodes(t, ctx, uploader, nodes) + return nil })) // load tree again - tree2, err := data.LoadTree(context.TODO(), repo, id) + it, err := data.LoadTree(context.TODO(), repo, id) rtest.OK(t, err) + nodes2 := []*data.Node{} + for item := range it { + rtest.OK(t, item.Error) + nodes2 = append(nodes2, item.Node) + } - rtest.Assert(t, tree.Equals(tree2), - "trees are not equal: want %v, got %v", - tree, tree2) + rtest.Assert(t, slices.Equal(nodes, nodes2), + "tree nodes are not equal: want %v, got %v", + nodes, nodes2) +} + +// Basic type for comparing the serialization of the tree +type Tree struct { + Nodes []*data.Node `json:"nodes"` } func TestTreeEqualSerialization(t *testing.T) { files := []string{"node.go", "tree.go", "tree_test.go"} for i := 1; i <= len(files); i++ { - tree := data.NewTree(i) + tree := Tree{Nodes: make([]*data.Node, 0, i)} builder := data.NewTreeJSONBuilder() for _, fn := range files[:i] { node := nodeForFile(t, fn) - rtest.OK(t, tree.Insert(node)) + tree.Nodes = append(tree.Nodes, node) rtest.OK(t, builder.AddNode(node)) - rtest.Assert(t, tree.Insert(node) != nil, "no error on duplicate node") rtest.Assert(t, builder.AddNode(node) != nil, "no error on duplicate node") rtest.Assert(t, errors.Is(builder.AddNode(node), data.ErrTreeNotOrdered), "wrong error returned") } @@ -144,14 +155,34 @@ func TestTreeEqualSerialization(t *testing.T) { treeBytes = append(treeBytes, '\n') rtest.OK(t, err) - stiBytes, err := builder.Finalize() + buf, err := builder.Finalize() rtest.OK(t, err) // compare serialization of an individual node and the SaveTreeIterator - rtest.Equals(t, treeBytes, stiBytes) + rtest.Equals(t, treeBytes, buf) } } +func TestTreeLoadSaveCycle(t *testing.T) { + files := []string{"node.go", "tree.go", "tree_test.go"} + builder := data.NewTreeJSONBuilder() + for _, fn := range files { + node := nodeForFile(t, fn) + rtest.OK(t, builder.AddNode(node)) + } + buf, err := builder.Finalize() + rtest.OK(t, err) + + tm := data.TestTreeMap{restic.Hash(buf): buf} + it, err := data.LoadTree(context.TODO(), tm, restic.Hash(buf)) + rtest.OK(t, err) + + mtm := data.TestWritableTreeMap{TestTreeMap: data.TestTreeMap{}} + id, err := data.SaveTree(context.TODO(), mtm, it) + rtest.OK(t, err) + rtest.Equals(t, restic.Hash(buf), id, "saved tree id mismatch") +} + func BenchmarkBuildTree(b *testing.B) { const size = 100 // Directories of this size are not uncommon. @@ -165,11 +196,12 @@ func BenchmarkBuildTree(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - t := data.NewTree(size) - + t := data.NewTreeJSONBuilder() for i := range nodes { - _ = t.Insert(&nodes[i]) + rtest.OK(b, t.AddNode(&nodes[i])) } + _, err := t.Finalize() + rtest.OK(b, err) } } @@ -186,8 +218,80 @@ func testLoadTree(t *testing.T, version uint) { repo, _, _ := repository.TestRepositoryWithVersion(t, version) sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) - _, err := data.LoadTree(context.TODO(), repo, *sn.Tree) + nodes, err := data.LoadTree(context.TODO(), repo, *sn.Tree) rtest.OK(t, err) + for item := range nodes { + rtest.OK(t, item.Error) + } +} + +func TestTreeIteratorUnknownKeys(t *testing.T) { + tests := []struct { + name string + jsonData string + wantNodes []string + }{ + { + name: "unknown key before nodes", + jsonData: `{"extra": "value", "nodes": [{"name": "test1"}, {"name": "test2"}]}`, + wantNodes: []string{"test1", "test2"}, + }, + { + name: "unknown key after nodes", + jsonData: `{"nodes": [{"name": "test1"}, {"name": "test2"}], "extra": "value"}`, + wantNodes: []string{"test1", "test2"}, + }, + { + name: "multiple unknown keys before nodes", + jsonData: `{"key1": "value1", "key2": 42, "nodes": [{"name": "test1"}]}`, + wantNodes: []string{"test1"}, + }, + { + name: "multiple unknown keys after nodes", + jsonData: `{"nodes": [{"name": "test1"}], "key1": "value1", "key2": 42}`, + wantNodes: []string{"test1"}, + }, + { + name: "unknown keys before and after nodes", + jsonData: `{"before": "value", "nodes": [{"name": "test1"}], "after": "value"}`, + wantNodes: []string{"test1"}, + }, + { + name: "nested object as unknown value", + jsonData: `{"extra": {"nested": "value"}, "nodes": [{"name": "test1"}]}`, + wantNodes: []string{"test1"}, + }, + { + name: "nested array as unknown value", + jsonData: `{"extra": [1, 2, 3], "nodes": [{"name": "test1"}]}`, + wantNodes: []string{"test1"}, + }, + { + name: "complex nested structure as unknown value", + jsonData: `{"extra": {"obj": {"arr": [1, {"nested": true}]}}, "nodes": [{"name": "test1"}]}`, + wantNodes: []string{"test1"}, + }, + { + name: "empty nodes array with unknown keys", + jsonData: `{"extra": "value", "nodes": []}`, + wantNodes: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + it, err := data.NewTreeNodeIterator(strings.NewReader(tt.jsonData + "\n")) + rtest.OK(t, err) + + var gotNodes []string + for item := range it { + rtest.OK(t, item.Error) + gotNodes = append(gotNodes, item.Node.Name) + } + + rtest.Equals(t, tt.wantNodes, gotNodes, "nodes mismatch") + }) + } } func BenchmarkLoadTree(t *testing.B) { @@ -211,6 +315,58 @@ func benchmarkLoadTree(t *testing.B, version uint) { } } +func TestTreeFinderNilIterator(t *testing.T) { + finder := data.NewTreeFinder(nil) + defer finder.Close() + node, err := finder.Find("foo") + rtest.OK(t, err) + rtest.Equals(t, node, nil, "finder should return nil node") +} + +func TestTreeFinderError(t *testing.T) { + testErr := errors.New("error") + finder := data.NewTreeFinder(slices.Values([]data.NodeOrError{ + {Node: &data.Node{Name: "a"}, Error: nil}, + {Node: &data.Node{Name: "b"}, Error: nil}, + {Node: nil, Error: testErr}, + })) + defer finder.Close() + node, err := finder.Find("b") + rtest.OK(t, err) + rtest.Equals(t, node.Name, "b", "finder should return node with name b") + + node, err = finder.Find("c") + rtest.Equals(t, err, testErr, "finder should return correcterror") + rtest.Equals(t, node, nil, "finder should return nil node") +} + +func TestTreeFinderNotFound(t *testing.T) { + finder := data.NewTreeFinder(slices.Values([]data.NodeOrError{ + {Node: &data.Node{Name: "a"}, Error: nil}, + })) + defer finder.Close() + node, err := finder.Find("b") + rtest.OK(t, err) + rtest.Equals(t, node, nil, "finder should return nil node") + // must also be ok multiple times + node, err = finder.Find("c") + rtest.OK(t, err) + rtest.Equals(t, node, nil, "finder should return nil node") +} + +func TestTreeFinderWrongOrder(t *testing.T) { + finder := data.NewTreeFinder(slices.Values([]data.NodeOrError{ + {Node: &data.Node{Name: "d"}, Error: nil}, + })) + defer finder.Close() + node, err := finder.Find("b") + rtest.OK(t, err) + rtest.Equals(t, node, nil, "finder should return nil node") + node, err = finder.Find("a") + rtest.Assert(t, strings.Contains(err.Error(), "is not greater than"), "unexpected error: %v", err) + rtest.Equals(t, node, nil, "finder should return nil node") +} + func TestFindTreeDirectory(t *testing.T) { repo := repository.TestRepository(t) sn := data.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3) @@ -220,15 +376,15 @@ func TestFindTreeDirectory(t *testing.T) { id restic.ID err error }{ - {"", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil}, - {"/", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil}, - {".", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil}, + {"", restic.TestParseID("8804a5505fc3012e7d08b2843e9bda1bf3dc7644f64b542470340e1b4059f09f"), nil}, + {"/", restic.TestParseID("8804a5505fc3012e7d08b2843e9bda1bf3dc7644f64b542470340e1b4059f09f"), nil}, + {".", restic.TestParseID("8804a5505fc3012e7d08b2843e9bda1bf3dc7644f64b542470340e1b4059f09f"), nil}, {"..", restic.ID{}, errors.New("path ..: not found")}, {"file-1", restic.ID{}, errors.New("path file-1: not a directory")}, - {"dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil}, - {"/dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil}, - {"dir-21/", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil}, - {"dir-21/dir-24", restic.TestParseID("74626b3fb2bd4b3e572b81a4059b3e912bcf2a8f69fecd9c187613b7173f13b1"), nil}, + {"dir-7", restic.TestParseID("1af51eb70cd4457d51db40d649bb75446a3eaa29b265916d411bb7ae971d4849"), nil}, + {"/dir-7", restic.TestParseID("1af51eb70cd4457d51db40d649bb75446a3eaa29b265916d411bb7ae971d4849"), nil}, + {"dir-7/", restic.TestParseID("1af51eb70cd4457d51db40d649bb75446a3eaa29b265916d411bb7ae971d4849"), nil}, + {"dir-7/dir-5", restic.TestParseID("f05534d2673964de698860e5069da1ee3c198acf21c187975c6feb49feb8e9c9"), nil}, } { t.Run("", func(t *testing.T) { id, err := data.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder) @@ -244,3 +400,187 @@ func TestFindTreeDirectory(t *testing.T) { _, err := data.FindTreeDirectory(context.TODO(), repo, nil, "") rtest.Assert(t, err != nil, "missing error on null tree id") } + +func TestDualTreeIterator(t *testing.T) { + testErr := errors.New("test error") + + tests := []struct { + name string + tree1 []data.NodeOrError + tree2 []data.NodeOrError + expected []data.DualTree + }{ + { + name: "both empty", + tree1: []data.NodeOrError{}, + tree2: []data.NodeOrError{}, + expected: []data.DualTree{}, + }, + { + name: "tree1 empty", + tree1: []data.NodeOrError{}, + tree2: []data.NodeOrError{ + {Node: &data.Node{Name: "a"}}, + {Node: &data.Node{Name: "b"}}, + }, + expected: []data.DualTree{ + {Tree1: nil, Tree2: &data.Node{Name: "a"}, Error: nil}, + {Tree1: nil, Tree2: &data.Node{Name: "b"}, Error: nil}, + }, + }, + { + name: "tree2 empty", + tree1: []data.NodeOrError{ + {Node: &data.Node{Name: "a"}}, + {Node: &data.Node{Name: "b"}}, + }, + tree2: []data.NodeOrError{}, + expected: []data.DualTree{ + {Tree1: &data.Node{Name: "a"}, Tree2: nil, Error: nil}, + {Tree1: &data.Node{Name: "b"}, Tree2: nil, Error: nil}, + }, + }, + { + name: "identical trees", + tree1: []data.NodeOrError{ + {Node: &data.Node{Name: "a"}}, + {Node: &data.Node{Name: "b"}}, + }, + tree2: []data.NodeOrError{ + {Node: &data.Node{Name: "a"}}, + {Node: &data.Node{Name: "b"}}, + }, + expected: []data.DualTree{ + {Tree1: &data.Node{Name: "a"}, Tree2: &data.Node{Name: "a"}, Error: nil}, + {Tree1: &data.Node{Name: "b"}, Tree2: &data.Node{Name: "b"}, Error: nil}, + }, + }, + { + name: "disjoint trees", + tree1: []data.NodeOrError{ + {Node: &data.Node{Name: "a"}}, + {Node: &data.Node{Name: "c"}}, + }, + tree2: []data.NodeOrError{ + {Node: &data.Node{Name: "b"}}, + {Node: &data.Node{Name: "d"}}, + }, + expected: []data.DualTree{ + {Tree1: &data.Node{Name: "a"}, Tree2: nil, Error: nil}, + {Tree1: nil, Tree2: &data.Node{Name: "b"}, Error: nil}, + {Tree1: &data.Node{Name: "c"}, Tree2: nil, Error: nil}, + {Tree1: nil, Tree2: &data.Node{Name: "d"}, Error: nil}, + }, + }, + { + name: "overlapping trees", + tree1: []data.NodeOrError{ + {Node: &data.Node{Name: "a"}}, + {Node: &data.Node{Name: "b"}}, + {Node: &data.Node{Name: "d"}}, + }, + tree2: []data.NodeOrError{ + {Node: &data.Node{Name: "b"}}, + {Node: &data.Node{Name: "c"}}, + {Node: &data.Node{Name: "d"}}, + }, + expected: []data.DualTree{ + {Tree1: &data.Node{Name: "a"}, Tree2: nil, Error: nil}, + {Tree1: &data.Node{Name: "b"}, Tree2: &data.Node{Name: "b"}, Error: nil}, + {Tree1: nil, Tree2: &data.Node{Name: "c"}, Error: nil}, + {Tree1: &data.Node{Name: "d"}, Tree2: &data.Node{Name: "d"}, Error: nil}, + }, + }, + { + name: "error in tree1 during iteration", + tree1: []data.NodeOrError{ + {Node: &data.Node{Name: "a"}}, + {Error: testErr}, + }, + tree2: []data.NodeOrError{ + {Node: &data.Node{Name: "c"}}, + }, + expected: []data.DualTree{ + {Tree1: nil, Tree2: nil, Error: testErr}, + }, + }, + { + name: "error in tree2 during iteration", + tree1: []data.NodeOrError{ + {Node: &data.Node{Name: "a"}}, + }, + tree2: []data.NodeOrError{ + {Node: &data.Node{Name: "b"}}, + {Error: testErr}, + }, + expected: []data.DualTree{ + {Tree1: &data.Node{Name: "a"}, Tree2: nil, Error: nil}, + {Tree1: nil, Tree2: nil, Error: testErr}, + }, + }, + { + name: "error at start of tree1", + tree1: []data.NodeOrError{{Error: testErr}}, + tree2: []data.NodeOrError{{Node: &data.Node{Name: "b"}}}, + expected: []data.DualTree{ + {Tree1: nil, Tree2: nil, Error: testErr}, + }, + }, + { + name: "error at start of tree2", + tree1: []data.NodeOrError{{Node: &data.Node{Name: "a"}}}, + tree2: []data.NodeOrError{{Error: testErr}}, + expected: []data.DualTree{ + {Tree1: nil, Tree2: nil, Error: testErr}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iter1 := slices.Values(tt.tree1) + iter2 := slices.Values(tt.tree2) + + dualIter := data.DualTreeIterator(iter1, iter2) + var results []data.DualTree + for dt := range dualIter { + results = append(results, dt) + } + + rtest.Equals(t, len(tt.expected), len(results), "unexpected number of results") + for i, exp := range tt.expected { + rtest.Equals(t, exp.Error, results[i].Error, fmt.Sprintf("error mismatch at index %d", i)) + rtest.Equals(t, exp.Tree1, results[i].Tree1, fmt.Sprintf("Tree1 mismatch at index %d", i)) + rtest.Equals(t, exp.Tree2, results[i].Tree2, fmt.Sprintf("Tree2 mismatch at index %d", i)) + } + }) + } + + t.Run("single use restriction", func(t *testing.T) { + iter1 := slices.Values([]data.NodeOrError{{Node: &data.Node{Name: "a"}}}) + iter2 := slices.Values([]data.NodeOrError{{Node: &data.Node{Name: "b"}}}) + dualIter := data.DualTreeIterator(iter1, iter2) + + // First use should work + var count int + for range dualIter { + count++ + } + rtest.Assert(t, count > 0, "first iteration should produce results") + + // Second use should panic + func() { + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic on second use") + } + }() + count = 0 + for range dualIter { + // Should panic before reaching here + count++ + } + rtest.Equals(t, count, 0, "expected count to be 0") + }() + }) +} diff --git a/internal/debug/round_tripper_debug.go b/internal/debug/round_tripper_debug.go index df207207b..f3b34ad42 100644 --- a/internal/debug/round_tripper_debug.go +++ b/internal/debug/round_tripper_debug.go @@ -1,5 +1,4 @@ //go:build debug -// +build debug package debug diff --git a/internal/debug/round_tripper_release.go b/internal/debug/round_tripper_release.go index 6edadb479..a42754c08 100644 --- a/internal/debug/round_tripper_release.go +++ b/internal/debug/round_tripper_release.go @@ -1,5 +1,4 @@ //go:build !debug -// +build !debug package debug diff --git a/internal/dump/common.go b/internal/dump/common.go index aea5c1291..2c0edf67a 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -30,33 +30,42 @@ func New(format string, repo restic.Loader, w io.Writer) *Dumper { } } -func (d *Dumper) DumpTree(ctx context.Context, tree *data.Tree, rootPath string) error { - ctx, cancel := context.WithCancel(ctx) - defer cancel() +func (d *Dumper) DumpTree(ctx context.Context, tree data.TreeNodeIterator, rootPath string) error { + wg, ctx := errgroup.WithContext(ctx) // ch is buffered to deal with variable download/write speeds. ch := make(chan *data.Node, 10) - go sendTrees(ctx, d.repo, tree, rootPath, ch) + wg.Go(func() error { + return sendTrees(ctx, d.repo, tree, rootPath, ch) + }) - switch d.format { - case "tar": - return d.dumpTar(ctx, ch) - case "zip": - return d.dumpZip(ctx, ch) - default: - panic("unknown dump format") - } + wg.Go(func() error { + switch d.format { + case "tar": + return d.dumpTar(ctx, ch) + case "zip": + return d.dumpZip(ctx, ch) + default: + panic("unknown dump format") + } + }) + return wg.Wait() } -func sendTrees(ctx context.Context, repo restic.BlobLoader, tree *data.Tree, rootPath string, ch chan *data.Node) { +func sendTrees(ctx context.Context, repo restic.BlobLoader, nodes data.TreeNodeIterator, rootPath string, ch chan *data.Node) error { defer close(ch) - for _, root := range tree.Nodes { - root.Path = path.Join(rootPath, root.Name) - if sendNodes(ctx, repo, root, ch) != nil { - break + for item := range nodes { + if item.Error != nil { + return item.Error + } + node := item.Node + node.Path = path.Join(rootPath, node.Name) + if err := sendNodes(ctx, repo, node, ch); err != nil { + return err } } + return nil } func sendNodes(ctx context.Context, repo restic.BlobLoader, root *data.Node, ch chan *data.Node) error { diff --git a/internal/dump/common_test.go b/internal/dump/common_test.go index 5599e2717..bb0347189 100644 --- a/internal/dump/common_test.go +++ b/internal/dump/common_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" @@ -13,13 +14,13 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (string, restic.Repository) { +func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (string, restic.Repository, backend.Backend) { tempdir := rtest.TempDir(t) - repo := repository.TestRepository(t) + repo, _, be := repository.TestRepositoryWithVersion(t, 0) archiver.TestCreateFiles(t, tempdir, src) - return tempdir, repo + return tempdir, repo, be } type CheckDump func(t *testing.T, testDir string, testDump *bytes.Buffer) error @@ -67,13 +68,22 @@ func WriteTest(t *testing.T, format string, cd CheckDump) { }, target: "/", }, + { + name: "directory only", + args: archiver.TestDir{ + "firstDir": archiver.TestDir{ + "secondDir": archiver.TestDir{}, + }, + }, + target: "/", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tmpdir, repo := prepareTempdirRepoSrc(t, tt.args) + tmpdir, repo, be := prepareTempdirRepoSrc(t, tt.args) arch := archiver.New(repo, fs.Track{FS: fs.Local{}}, archiver.Options{}) back := rtest.Chdir(t, tmpdir) @@ -93,6 +103,15 @@ func WriteTest(t *testing.T, format string, cd CheckDump) { if err := cd(t, tmpdir, dst); err != nil { t.Errorf("WriteDump() = does not match: %v", err) } + + // test that dump returns an error if the repository is broken + tree, err = data.LoadTree(ctx, repo, *sn.Tree) + rtest.OK(t, err) + rtest.OK(t, be.Delete(ctx)) + // use new dumper as the old one has the blobs cached + d = New(format, repo, dst) + err = d.DumpTree(ctx, tree, tt.target) + rtest.Assert(t, err != nil, "expected error, got nil") }) } } diff --git a/internal/filter/include.go b/internal/filter/include.go index 87d5f1207..e6eefe6b9 100644 --- a/internal/filter/include.go +++ b/internal/filter/include.go @@ -25,6 +25,10 @@ func (opts *IncludePatternOptions) Add(f *pflag.FlagSet) { f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns") } +func (opts *IncludePatternOptions) Empty() bool { + return len(opts.Includes) == 0 && len(opts.InsensitiveIncludes) == 0 && len(opts.IncludeFiles) == 0 && len(opts.InsensitiveIncludeFiles) == 0 +} + func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]IncludeByNameFunc, error) { var fs []IncludeByNameFunc if len(opts.IncludeFiles) > 0 { diff --git a/internal/fs/const_unix.go b/internal/fs/const_unix.go index e570c2553..b75387394 100644 --- a/internal/fs/const_unix.go +++ b/internal/fs/const_unix.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package fs diff --git a/internal/fs/const_windows.go b/internal/fs/const_windows.go index b2b1bab86..8e7aebf6c 100644 --- a/internal/fs/const_windows.go +++ b/internal/fs/const_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fs/ea_windows.go b/internal/fs/ea_windows.go index e4c32057d..43bccd504 100644 --- a/internal/fs/ea_windows.go +++ b/internal/fs/ea_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fs/ea_windows_test.go b/internal/fs/ea_windows_test.go index 00cbe97f8..c61ea427d 100644 --- a/internal/fs/ea_windows_test.go +++ b/internal/fs/ea_windows_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fs/file_unix.go b/internal/fs/file_unix.go index 4e7765c30..9b33f1ab7 100644 --- a/internal/fs/file_unix.go +++ b/internal/fs/file_unix.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package fs diff --git a/internal/fs/fs_local_vss_test.go b/internal/fs/fs_local_vss_test.go index bae08bd10..9628cc4ef 100644 --- a/internal/fs/fs_local_vss_test.go +++ b/internal/fs/fs_local_vss_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fs/mknod_unix.go b/internal/fs/mknod_unix.go index 024c4d502..1ee6509b1 100644 --- a/internal/fs/mknod_unix.go +++ b/internal/fs/mknod_unix.go @@ -1,5 +1,4 @@ //go:build !freebsd && !windows -// +build !freebsd,!windows package fs diff --git a/internal/fs/node_freebsd.go b/internal/fs/node_freebsd.go index 0cbe876f1..37caba8f0 100644 --- a/internal/fs/node_freebsd.go +++ b/internal/fs/node_freebsd.go @@ -1,5 +1,4 @@ //go:build freebsd -// +build freebsd package fs diff --git a/internal/fs/node_noxattr.go b/internal/fs/node_noxattr.go index c061d1777..3e3cdbd78 100644 --- a/internal/fs/node_noxattr.go +++ b/internal/fs/node_noxattr.go @@ -1,5 +1,4 @@ //go:build aix || dragonfly || openbsd -// +build aix dragonfly openbsd package fs diff --git a/internal/fs/node_unix.go b/internal/fs/node_unix.go index 0b43327c4..539b220db 100644 --- a/internal/fs/node_unix.go +++ b/internal/fs/node_unix.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package fs diff --git a/internal/fs/node_unix_test.go b/internal/fs/node_unix_test.go index 85aad2b48..4489f8f12 100644 --- a/internal/fs/node_unix_test.go +++ b/internal/fs/node_unix_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package fs diff --git a/internal/fs/node_windows_test.go b/internal/fs/node_windows_test.go index 92d841d83..06c7bd4ef 100644 --- a/internal/fs/node_windows_test.go +++ b/internal/fs/node_windows_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs @@ -56,6 +55,176 @@ func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir string, file compareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath) } +// TestRestoreSecurityDescriptorInheritance checks that the DACL protection (inheritance) +// flags are correctly restored. This is the mechanism that preserves the `IsInherited` +// property on individual ACEs. +func TestRestoreSecurityDescriptorInheritance(t *testing.T) { + // This test requires admin privileges to manipulate ACLs effectively. + isAdmin, err := isAdmin() + test.OK(t, err) + if !isAdmin { + t.Skip("Skipping inheritance test, requires admin privileges") + } + + tempDir := t.TempDir() + + // 1. Create a parent/child directory structure. + parentDir := filepath.Join(tempDir, "parent") + err = os.Mkdir(parentDir, 0755) + test.OK(t, err) + + childDir := filepath.Join(parentDir, "child") + err = os.Mkdir(childDir, 0755) + test.OK(t, err) + + // 2. Set inheritable permissions on the parent. + // We will give the "Users" group inheritable read access. + users, err := windows.StringToSid("S-1-5-32-545") // BUILTIN\Users + test.OK(t, err) + + // Create an EXPLICIT_ACCESS structure for the new ACE. + explicitAccess := windows.EXPLICIT_ACCESS{ + AccessPermissions: windows.GENERIC_READ, + AccessMode: windows.GRANT_ACCESS, + Inheritance: windows.OBJECT_INHERIT_ACE | windows.CONTAINER_INHERIT_ACE, + Trustee: windows.TRUSTEE{ + TrusteeForm: windows.TRUSTEE_IS_SID, + TrusteeType: windows.TRUSTEE_IS_GROUP, + TrusteeValue: windows.TrusteeValueFromSID(users), + }, + } + + // Create a new DACL from the entry. + dacl, err := windows.ACLFromEntries([]windows.EXPLICIT_ACCESS{explicitAccess}, nil) + test.OK(t, err) + + // Apply this new DACL to the parent, marking it as unprotected so it can be inherited. + err = windows.SetNamedSecurityInfo( + parentDir, + windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION|windows.UNPROTECTED_DACL_SECURITY_INFORMATION, + nil, nil, dacl, nil, + ) + test.OK(t, errors.Wrapf(err, "failed to set inheritable ACL on parent dir")) + + // 3. Get the Security Descriptor of the child, which should now have inherited the ACE. + sdBytesOriginal, err := getSecurityDescriptor(childDir) + test.OK(t, err) + + // Sanity check: verify the original child SD is NOT protected from inheritance. + sdOriginal, err := securityDescriptorBytesToStruct(*sdBytesOriginal) + test.OK(t, err) + control, _, err := sdOriginal.Control() + test.OK(t, err) + test.Assert(t, control&windows.SE_DACL_PROTECTED == 0, "Pre-condition failed: child directory should have inheritance enabled") + + // 4. Create a restic node for the child directory. + genericAttrs, err := data.WindowsAttrsToGenericAttributes(data.WindowsAttributes{SecurityDescriptor: sdBytesOriginal}) + test.OK(t, err) + childNode := getNode("child-restored", "dir", genericAttrs) + + // 5. Restore the node to a new location. + restoreDir := filepath.Join(tempDir, "restore") + err = os.Mkdir(restoreDir, 0755) + test.OK(t, err) + + restoredPath, _ := restoreAndGetNode(t, restoreDir, &childNode, false) + + // 6. Get the Security Descriptor of the restored child directory. + sdBytesRestored, err := getSecurityDescriptor(restoredPath) + test.OK(t, err) + + // 7. Compare the control flags of the original and restored SDs. + sdRestored, err := securityDescriptorBytesToStruct(*sdBytesRestored) + test.OK(t, err) + controlRestored, _, err := sdRestored.Control() + test.OK(t, err) + + // The core of the test: Ensure the restored DACL protection flag matches the original. + originalIsProtected := (control & windows.SE_DACL_PROTECTED) != 0 + restoredIsProtected := (controlRestored & windows.SE_DACL_PROTECTED) != 0 + test.Equals(t, originalIsProtected, restoredIsProtected, "DACL protection flag was not restored correctly. Inheritance state is wrong.") +} + +// TestRestoreSecurityDescriptorInheritanceLowPrivilege tests that the low-privilege restore +// path (setNamedSecurityInfoLow) correctly handles inheritance flags. This test doesn't require +// admin privileges and focuses on DACL restoration only. +func TestRestoreSecurityDescriptorInheritanceLowPrivilege(t *testing.T) { + tempDir := t.TempDir() + + // 1. Create a test directory + testDir := filepath.Join(tempDir, "testdir") + err := os.Mkdir(testDir, 0755) + test.OK(t, err) + + // 2. Get its security descriptor (which will have some default ACL) + sdBytesOriginal, err := getSecurityDescriptor(testDir) + test.OK(t, err) + + // Verify we can get the control flags + sdOriginal, err := securityDescriptorBytesToStruct(*sdBytesOriginal) + test.OK(t, err) + controlOriginal, _, err := sdOriginal.Control() + test.OK(t, err) + + // 3. Test both protected and unprotected scenarios by modifying the control flags + testCases := []struct { + name string + shouldBeProtected bool + }{ + {"unprotected_inheritance_enabled", false}, + {"protected_inheritance_disabled", true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create a copy of the SD bytes to modify + sdBytesTest := make([]byte, len(*sdBytesOriginal)) + copy(sdBytesTest, *sdBytesOriginal) + + // Modify the control flags to simulate protected/unprotected DACL + sdTest, err := securityDescriptorBytesToStruct(sdBytesTest) + test.OK(t, err) + + // Get the DACL from the test SD + dacl, _, err := sdTest.DACL() + test.OK(t, err) + + // Determine which control flag to use based on test case + var controlToUse windows.SECURITY_DESCRIPTOR_CONTROL + if tc.shouldBeProtected { + controlToUse = controlOriginal | windows.SE_DACL_PROTECTED + } else { + controlToUse = controlOriginal &^ windows.SE_DACL_PROTECTED + } + + // 4. Call setNamedSecurityInfoLow directly to test the low-privilege path + restoreTarget := filepath.Join(tempDir, "restore_"+tc.name) + err = os.Mkdir(restoreTarget, 0755) + test.OK(t, err) + + // This directly tests the low-privilege restore function + err = setNamedSecurityInfoLow(restoreTarget, dacl, controlToUse) + test.OK(t, err) + + // 5. Get the security descriptor of the restored directory + sdBytesRestored, err := getSecurityDescriptor(restoreTarget) + test.OK(t, err) + + // 6. Verify that the control flags were correctly restored + sdRestored, err := securityDescriptorBytesToStruct(*sdBytesRestored) + test.OK(t, err) + controlRestored, _, err := sdRestored.Control() + test.OK(t, err) // Check if the protection flag matches what we requested + restoredIsProtected := (controlRestored & windows.SE_DACL_PROTECTED) != 0 + if tc.shouldBeProtected != restoredIsProtected { + t.Errorf("DACL protection flag was not restored correctly in low-privilege path. Expected protected=%v, got protected=%v", + tc.shouldBeProtected, restoredIsProtected) + } + }) + } +} + func getNode(name string, fileType data.NodeType, genericAttributes map[data.GenericAttributeType]json.RawMessage) data.Node { return data.Node{ Name: name, diff --git a/internal/fs/node_xattr.go b/internal/fs/node_xattr.go index 546df1f37..d6289c74a 100644 --- a/internal/fs/node_xattr.go +++ b/internal/fs/node_xattr.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || netbsd || linux || solaris -// +build darwin freebsd netbsd linux solaris package fs diff --git a/internal/fs/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go index dd26c97b3..0a2a62c45 100644 --- a/internal/fs/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || netbsd || linux || solaris || windows -// +build darwin freebsd netbsd linux solaris windows package fs diff --git a/internal/fs/node_xattr_test.go b/internal/fs/node_xattr_test.go index 7205e1fbe..65aeb7d6d 100644 --- a/internal/fs/node_xattr_test.go +++ b/internal/fs/node_xattr_test.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || netbsd || linux || solaris -// +build darwin freebsd netbsd linux solaris package fs diff --git a/internal/fs/preallocate_other.go b/internal/fs/preallocate_other.go index 4fb44d421..b04d68738 100644 --- a/internal/fs/preallocate_other.go +++ b/internal/fs/preallocate_other.go @@ -1,5 +1,4 @@ //go:build !linux && !darwin -// +build !linux,!darwin package fs diff --git a/internal/fs/priv.go b/internal/fs/priv.go index fc0089827..8cf8f4380 100644 --- a/internal/fs/priv.go +++ b/internal/fs/priv.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package fs diff --git a/internal/fs/priv_windows.go b/internal/fs/priv_windows.go index 5268951e2..e78f61c1c 100644 --- a/internal/fs/priv_windows.go +++ b/internal/fs/priv_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fs/priv_windows_test.go b/internal/fs/priv_windows_test.go index 7851194a6..0db7144e8 100644 --- a/internal/fs/priv_windows_test.go +++ b/internal/fs/priv_windows_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fs/sd_windows.go b/internal/fs/sd_windows.go index 1a1ee6d14..19966906a 100644 --- a/internal/fs/sd_windows.go +++ b/internal/fs/sd_windows.go @@ -12,14 +12,17 @@ import ( var lowerPrivileges atomic.Bool -// Flags for backup and restore with admin permissions -var highSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.SACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.BACKUP_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION | windows.PROTECTED_SACL_SECURITY_INFORMATION | windows.UNPROTECTED_DACL_SECURITY_INFORMATION | windows.UNPROTECTED_SACL_SECURITY_INFORMATION +// Flags for backup with admin permissions. Includes protection flags for GET operations. +var highBackupSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.SACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.BACKUP_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION | windows.PROTECTED_SACL_SECURITY_INFORMATION | windows.UNPROTECTED_DACL_SECURITY_INFORMATION | windows.UNPROTECTED_SACL_SECURITY_INFORMATION + +// Flags for restore with admin permissions. Base flags without protection flags for SET operations. +var highRestoreSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.SACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.BACKUP_SECURITY_INFORMATION // Flags for backup without admin permissions. If there are no admin permissions, only the current user's owner, group and DACL will be backed up. var lowBackupSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION | windows.UNPROTECTED_DACL_SECURITY_INFORMATION // Flags for restore without admin permissions. If there are no admin permissions, only the DACL from the SD can be restored and owner and group will be set based on the current user. -var lowRestoreSecurityFlags windows.SECURITY_INFORMATION = windows.DACL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION +var lowRestoreSecurityFlags windows.SECURITY_INFORMATION = windows.DACL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION // getSecurityDescriptor takes the path of the file and returns the SecurityDescriptor for the file. // This needs admin permissions or SeBackupPrivilege for getting the full SD. @@ -95,15 +98,21 @@ func setSecurityDescriptor(filePath string, securityDescriptor *[]byte) error { sacl = nil } + // Get the control flags from the original security descriptor + control, _, err := sd.Control() + if err != nil { + // This is unlikely to fail if the sd is valid, but handle it. + return fmt.Errorf("could not get security descriptor control flags: %w", err) + } // store original value to avoid unrelated changes in the error check useLowerPrivileges := lowerPrivileges.Load() if useLowerPrivileges { - err = setNamedSecurityInfoLow(filePath, dacl) + err = setNamedSecurityInfoLow(filePath, dacl, control) } else { - err = setNamedSecurityInfoHigh(filePath, owner, group, dacl, sacl) + err = setNamedSecurityInfoHigh(filePath, owner, group, dacl, sacl, control) // See corresponding fallback in getSecurityDescriptor for an explanation if err != nil && isAccessDeniedError(err) { - err = setNamedSecurityInfoLow(filePath, dacl) + err = setNamedSecurityInfoLow(filePath, dacl, control) } } @@ -121,7 +130,7 @@ func setSecurityDescriptor(filePath string, securityDescriptor *[]byte) error { // getNamedSecurityInfoHigh gets the higher level SecurityDescriptor which requires admin permissions. func getNamedSecurityInfoHigh(filePath string) (*windows.SECURITY_DESCRIPTOR, error) { - return windows.GetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, highSecurityFlags) + return windows.GetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, highBackupSecurityFlags) } // getNamedSecurityInfoLow gets the lower level SecurityDescriptor which requires no admin permissions. @@ -130,13 +139,40 @@ func getNamedSecurityInfoLow(filePath string) (*windows.SECURITY_DESCRIPTOR, err } // setNamedSecurityInfoHigh sets the higher level SecurityDescriptor which requires admin permissions. -func setNamedSecurityInfoHigh(filePath string, owner *windows.SID, group *windows.SID, dacl *windows.ACL, sacl *windows.ACL) error { - return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, highSecurityFlags, owner, group, dacl, sacl) +func setNamedSecurityInfoHigh(filePath string, owner *windows.SID, group *windows.SID, dacl *windows.ACL, sacl *windows.ACL, control windows.SECURITY_DESCRIPTOR_CONTROL) error { + securityInfo := highRestoreSecurityFlags + + // Check if the original DACL was protected from inheritance and add the correct flag. + if control&windows.SE_DACL_PROTECTED != 0 { + securityInfo |= windows.PROTECTED_DACL_SECURITY_INFORMATION + } else { + // Explicitly state that it is NOT protected. This ensures inheritance is re-enabled correctly. + securityInfo |= windows.UNPROTECTED_DACL_SECURITY_INFORMATION + } + + // Do the same for the SACL for completeness. + if control&windows.SE_SACL_PROTECTED != 0 { + securityInfo |= windows.PROTECTED_SACL_SECURITY_INFORMATION + } else { + securityInfo |= windows.UNPROTECTED_SACL_SECURITY_INFORMATION + } + + return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, securityInfo, owner, group, dacl, sacl) } // setNamedSecurityInfoLow sets the lower level SecurityDescriptor which requires no admin permissions. -func setNamedSecurityInfoLow(filePath string, dacl *windows.ACL) error { - return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, lowRestoreSecurityFlags, nil, nil, dacl, nil) +func setNamedSecurityInfoLow(filePath string, dacl *windows.ACL, control windows.SECURITY_DESCRIPTOR_CONTROL) error { + securityInfo := lowRestoreSecurityFlags + + // Check if the original DACL was protected from inheritance and add the correct flag. + if control&windows.SE_DACL_PROTECTED != 0 { + securityInfo |= windows.PROTECTED_DACL_SECURITY_INFORMATION + } else { + // Explicitly state that it is NOT protected. This ensures inheritance is re-enabled correctly. + securityInfo |= windows.UNPROTECTED_DACL_SECURITY_INFORMATION + } + + return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, securityInfo, nil, nil, dacl, nil) } // isHandlePrivilegeNotHeldError checks if the error is ERROR_PRIVILEGE_NOT_HELD diff --git a/internal/fs/sd_windows_test.go b/internal/fs/sd_windows_test.go index c31b19b8b..56e5fde07 100644 --- a/internal/fs/sd_windows_test.go +++ b/internal/fs/sd_windows_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fs/sd_windows_test_helpers.go b/internal/fs/sd_windows_test_helpers.go index 0e888884a..14f3ead3a 100644 --- a/internal/fs/sd_windows_test_helpers.go +++ b/internal/fs/sd_windows_test_helpers.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fs/setflags_other.go b/internal/fs/setflags_other.go index 281aef093..2568181b4 100644 --- a/internal/fs/setflags_other.go +++ b/internal/fs/setflags_other.go @@ -1,5 +1,4 @@ //go:build !linux -// +build !linux package fs diff --git a/internal/fs/stat_bsd.go b/internal/fs/stat_bsd.go index 5a3704194..84155c82b 100644 --- a/internal/fs/stat_bsd.go +++ b/internal/fs/stat_bsd.go @@ -1,5 +1,4 @@ //go:build freebsd || netbsd -// +build freebsd netbsd package fs diff --git a/internal/fs/stat_darwin.go b/internal/fs/stat_darwin.go index ee39e8bfa..ec94d9f8b 100644 --- a/internal/fs/stat_darwin.go +++ b/internal/fs/stat_darwin.go @@ -1,5 +1,4 @@ //go:build darwin -// +build darwin package fs diff --git a/internal/fs/stat_unix.go b/internal/fs/stat_unix.go index 70124658f..176311f7e 100644 --- a/internal/fs/stat_unix.go +++ b/internal/fs/stat_unix.go @@ -1,5 +1,4 @@ //go:build !windows && !darwin && !freebsd && !netbsd -// +build !windows,!darwin,!freebsd,!netbsd package fs diff --git a/internal/fs/stat_windows.go b/internal/fs/stat_windows.go index a62ddf87f..bc4092df8 100644 --- a/internal/fs/stat_windows.go +++ b/internal/fs/stat_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fs/vss.go b/internal/fs/vss.go index 3215c9aa3..93b768dbc 100644 --- a/internal/fs/vss.go +++ b/internal/fs/vss.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package fs diff --git a/internal/fs/vss_windows.go b/internal/fs/vss_windows.go index dabe78fb4..874953c5a 100644 --- a/internal/fs/vss_windows.go +++ b/internal/fs/vss_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fs diff --git a/internal/fuse/dir.go b/internal/fuse/dir.go index 75fee3d17..df558ac1f 100644 --- a/internal/fuse/dir.go +++ b/internal/fuse/dir.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse @@ -8,6 +7,7 @@ import ( "errors" "os" "path/filepath" + "slices" "sync" "syscall" @@ -66,13 +66,13 @@ func unwrapCtxCanceled(err error) error { // replaceSpecialNodes replaces nodes with name "." and "/" by their contents. // Otherwise, the node is returned. -func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data.Node) ([]*data.Node, error) { +func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data.Node) (data.TreeNodeIterator, error) { if node.Type != data.NodeTypeDir || node.Subtree == nil { - return []*data.Node{node}, nil + return slices.Values([]data.NodeOrError{{Node: node}}), nil } if node.Name != "." && node.Name != "/" { - return []*data.Node{node}, nil + return slices.Values([]data.NodeOrError{{Node: node}}), nil } tree, err := data.LoadTree(ctx, repo, *node.Subtree) @@ -80,7 +80,7 @@ func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data return nil, unwrapCtxCanceled(err) } - return tree.Nodes, nil + return tree, nil } func newDirFromSnapshot(root *Root, forget forgetFn, inode uint64, snapshot *data.Snapshot) (*dir, error) { @@ -116,18 +116,25 @@ func (d *dir) open(ctx context.Context) error { return unwrapCtxCanceled(err) } items := make(map[string]*data.Node) - for _, n := range tree.Nodes { + for item := range tree { + if item.Error != nil { + return unwrapCtxCanceled(item.Error) + } if ctx.Err() != nil { return ctx.Err() } + n := item.Node nodes, err := replaceSpecialNodes(ctx, d.root.repo, n) if err != nil { debug.Log(" replaceSpecialNodes(%v) failed: %v", n, err) return err } - for _, node := range nodes { - items[cleanupNodeName(node.Name)] = node + for item := range nodes { + if item.Error != nil { + return unwrapCtxCanceled(item.Error) + } + items[cleanupNodeName(item.Node.Name)] = item.Node } } d.items = items diff --git a/internal/fuse/file.go b/internal/fuse/file.go index 8ce90961f..42a83d652 100644 --- a/internal/fuse/file.go +++ b/internal/fuse/file.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index dc8128b65..c82252458 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse @@ -60,7 +59,7 @@ func loadFirstSnapshot(t testing.TB, repo restic.ListerLoaderUnpacked) *data.Sna return sn } -func loadTree(t testing.TB, repo restic.Loader, id restic.ID) *data.Tree { +func loadTree(t testing.TB, repo restic.Loader, id restic.ID) data.TreeNodeIterator { tree, err := data.LoadTree(context.TODO(), repo, id) rtest.OK(t, err) return tree @@ -80,8 +79,9 @@ func TestFuseFile(t *testing.T) { tree := loadTree(t, repo, *sn.Tree) var content restic.IDs - for _, node := range tree.Nodes { - content = append(content, node.Content...) + for item := range tree { + rtest.OK(t, item.Error) + content = append(content, item.Node.Content...) } t.Logf("tree loaded, content: %v", content) diff --git a/internal/fuse/inode.go b/internal/fuse/inode.go index b70180445..d334c8d73 100644 --- a/internal/fuse/inode.go +++ b/internal/fuse/inode.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/fuse/link.go b/internal/fuse/link.go index e37cb48f3..5f545cd8c 100644 --- a/internal/fuse/link.go +++ b/internal/fuse/link.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/fuse/other.go b/internal/fuse/other.go index 07ba0fe50..b9cba74a0 100644 --- a/internal/fuse/other.go +++ b/internal/fuse/other.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/fuse/root.go b/internal/fuse/root.go index 40f52ae5a..6b02238dd 100644 --- a/internal/fuse/root.go +++ b/internal/fuse/root.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 4883b9ed8..645249417 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index 141132abc..e9b7e7aa6 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/fuse/snapshots_dirstruct_test.go b/internal/fuse/snapshots_dirstruct_test.go index 0295bca13..bd04af287 100644 --- a/internal/fuse/snapshots_dirstruct_test.go +++ b/internal/fuse/snapshots_dirstruct_test.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/fuse/tree_cache.go b/internal/fuse/tree_cache.go index d913f9b81..4b60aeb11 100644 --- a/internal/fuse/tree_cache.go +++ b/internal/fuse/tree_cache.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/fuse/xattr.go b/internal/fuse/xattr.go index f219c2a4b..26753e442 100644 --- a/internal/fuse/xattr.go +++ b/internal/fuse/xattr.go @@ -1,5 +1,4 @@ //go:build darwin || freebsd || linux -// +build darwin freebsd linux package fuse diff --git a/internal/global/global.go b/internal/global/global.go index c65338b92..88439462a 100644 --- a/internal/global/global.go +++ b/internal/global/global.go @@ -81,6 +81,9 @@ type Options struct { Options []string Extended options.Options + + // packSizeFlag is used to detect if --pack-size was set (CLI overrides env). + packSizeFlag *pflag.Flag } func (opts *Options) AddFlags(f *pflag.FlagSet) { @@ -106,7 +109,8 @@ func (opts *Options) AddFlags(f *pflag.FlagSet) { f.BoolVar(&opts.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)") f.IntVar(&opts.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&opts.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") - f.UintVar(&opts.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") + const packSizeFlag = "pack-size" + f.UintVar(&opts.PackSize, packSizeFlag, 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") f.StringSliceVarP(&opts.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") f.StringVar(&opts.HTTPUserAgent, "http-user-agent", "", "set a http user agent for outgoing http requests") f.DurationVar(&opts.StuckRequestTimeout, "stuck-request-timeout", 5*time.Minute, "`duration` after which to retry stuck requests") @@ -125,9 +129,7 @@ func (opts *Options) AddFlags(f *pflag.FlagSet) { // ignore error as there's no good way to handle it _ = opts.Compression.Set(comp) } - // parse target pack size from env, on error the default value will be used - targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32) - opts.PackSize = uint(targetPackSize) + opts.packSizeFlag = f.Lookup(packSizeFlag) if os.Getenv("RESTIC_HTTP_USER_AGENT") != "" { opts.HTTPUserAgent = os.Getenv("RESTIC_HTTP_USER_AGENT") @@ -135,6 +137,15 @@ func (opts *Options) AddFlags(f *pflag.FlagSet) { } func (opts *Options) PreRun(needsPassword bool) error { + if envVal := os.Getenv("RESTIC_PACK_SIZE"); envVal != "" && !opts.packSizeFlag.Changed { + targetPackSize, err := strconv.ParseUint(envVal, 10, 32) + if err != nil { + // Failing fast here keeps backups from running for a long time with the wrong pack size. + return errors.Fatalf("invalid value for RESTIC_PACK_SIZE %q: %v", envVal, err) + } + opts.PackSize = uint(targetPackSize) + } + // set verbosity, default is one opts.Verbosity = 1 if opts.Quiet && opts.Verbose > 0 { diff --git a/internal/global/global_debug.go b/internal/global/global_debug.go index be414ff80..c26009bac 100644 --- a/internal/global/global_debug.go +++ b/internal/global/global_debug.go @@ -1,5 +1,4 @@ //go:build debug || profile -// +build debug profile package global diff --git a/internal/global/global_release.go b/internal/global/global_release.go index f0c21ac60..a04ad0588 100644 --- a/internal/global/global_release.go +++ b/internal/global/global_release.go @@ -1,5 +1,4 @@ //go:build !debug && !profile -// +build !debug,!profile package global diff --git a/internal/global/global_test.go b/internal/global/global_test.go index 7d5ead722..474d9d0e1 100644 --- a/internal/global/global_test.go +++ b/internal/global/global_test.go @@ -7,6 +7,9 @@ import ( "strings" "testing" + "github.com/spf13/pflag" + + "github.com/restic/restic/internal/errors" rtest "github.com/restic/restic/internal/test" ) @@ -49,3 +52,41 @@ func TestReadEmptyPassword(t *testing.T) { _, err = readPassword(context.TODO(), opts, "test") rtest.Assert(t, strings.Contains(err.Error(), "must not be specified together with providing a password via a cli option or environment variable"), "unexpected error message, got %v", err) } + +func TestPackSizeEnvParseError(t *testing.T) { + t.Setenv("RESTIC_PACK_SIZE", "64MiB") + + var gopts Options + gopts.AddFlags(pflag.NewFlagSet("test", pflag.ContinueOnError)) + + err := gopts.PreRun(false) + rtest.Assert(t, err != nil, "expected error for invalid pack size env") + rtest.Assert(t, errors.IsFatal(err), "expected fatal error for invalid pack size env, got %T", err) + rtest.Assert(t, strings.Contains(err.Error(), "RESTIC_PACK_SIZE"), "error should mention RESTIC_PACK_SIZE, got %v", err) +} + +func TestPackSizeEnvApplied(t *testing.T) { + t.Setenv("RESTIC_PACK_SIZE", "64") + + var gopts Options + gopts.AddFlags(pflag.NewFlagSet("test", pflag.ContinueOnError)) + + err := gopts.PreRun(false) + rtest.OK(t, err) + rtest.Equals(t, uint(64), gopts.PackSize) +} + +func TestPackSizeEnvIgnoredWhenFlagSet(t *testing.T) { + t.Setenv("RESTIC_PACK_SIZE", "64MiB") + + var gopts Options + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + gopts.AddFlags(fs) + + err := fs.Set("pack-size", "64") + rtest.OK(t, err) + + err = gopts.PreRun(false) + rtest.OK(t, err) + rtest.Equals(t, uint(64), gopts.PackSize) +} diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go index 16155f3a4..62dbd167e 100644 --- a/internal/repository/fuzz_test.go +++ b/internal/repository/fuzz_test.go @@ -20,7 +20,7 @@ func FuzzSaveLoadBlob(f *testing.F) { id := restic.Hash(blob) repo, _, _ := TestRepositoryWithVersion(t, 2) - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { _, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, blob, id, false) return err })) diff --git a/internal/repository/index/associated_data.go b/internal/repository/index/associated_data.go index ad9b3288e..1267bdc4f 100644 --- a/internal/repository/index/associated_data.go +++ b/internal/repository/index/associated_data.go @@ -108,6 +108,40 @@ func (a *AssociatedSet[T]) Delete(bh restic.BlobHandle) { } } +type haser interface { + Has(bh restic.BlobHandle) bool +} + +// Intersect returns a new set containing the handles that are present in both sets. +func (a *AssociatedSet[T]) Intersect(other haser) *AssociatedSet[T] { + result := NewAssociatedSet[T](a.idx) + // Determining the smaller set already requires iterating over all keys + // and thus provides no performance benefit. + for bh := range a.Keys() { + if other.Has(bh) { + // preserve value receiver + val, _ := a.Get(bh) + result.Set(bh, val) + } + } + + return result +} + +// Sub returns a new set containing all handles that are present in a but not in +// other. +func (a *AssociatedSet[T]) Sub(other haser) *AssociatedSet[T] { + result := NewAssociatedSet[T](a.idx) + for bh := range a.Keys() { + if !other.Has(bh) { + val, _ := a.Get(bh) + result.Set(bh, val) + } + } + + return result +} + func (a *AssociatedSet[T]) Len() int { count := 0 for range a.All() { diff --git a/internal/repository/index/associated_data_test.go b/internal/repository/index/associated_data_test.go index 6d70b3dff..2d4611f5c 100644 --- a/internal/repository/index/associated_data_test.go +++ b/internal/repository/index/associated_data_test.go @@ -157,3 +157,81 @@ func TestAssociatedSetWithExtendedIndex(t *testing.T) { test.Equals(t, list(bs), restic.BlobHandles(nil)) test.Equals(t, 0, len(bs.overflow)) } + +func TestAssociatedSetIntersectAndSub(t *testing.T) { + mi := NewMasterIndex() + saver := &noopSaver{} + + bh1, blob1 := makeFakePackedBlob() + bh2, blob2 := makeFakePackedBlob() + bh3, blob3 := makeFakePackedBlob() + bh4, blob4 := makeFakePackedBlob() + + test.OK(t, mi.StorePack(context.TODO(), blob1.PackID, []restic.Blob{blob1.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, []restic.Blob{blob2.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob3.PackID, []restic.Blob{blob3.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob4.PackID, []restic.Blob{blob4.Blob}, saver)) + test.OK(t, mi.Flush(context.TODO(), saver)) + + t.Run("Intersect", func(t *testing.T) { + bs1, bs2 := NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + test.Equals(t, bs1.Intersect(bs2).Len(), 0) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh1, 10) + bs2.Set(bh2, 20) + test.Equals(t, bs1.Intersect(bs2).Len(), 0) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh3, 40) + bs2.Set(bh3, 50) + bs2.Set(bh4, 60) + result := bs1.Intersect(bs2) + test.Equals(t, result.Len(), 1) + val, _ := result.Get(bh3) + test.Equals(t, uint8(40), val) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh3, 40) + bs1.Set(bh4, 70) + bs2.Set(bh3, 50) + bs2.Set(bh4, 60) + result = bs1.Intersect(bs2) + test.Equals(t, result.Len(), 2) + val, _ = result.Get(bh3) + test.Equals(t, uint8(40), val) + val, _ = result.Get(bh4) + test.Equals(t, uint8(70), val) + }) + + t.Run("Sub", func(t *testing.T) { + bs1, bs2 := NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + test.Equals(t, bs1.Sub(bs2).Len(), 0) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh1, 10) + bs1.Set(bh2, 20) + bs2.Set(bh3, 30) + result := bs1.Sub(bs2) + test.Equals(t, result.Len(), 2) + val, _ := result.Get(bh1) + test.Equals(t, uint8(10), val) + val, _ = result.Get(bh2) + test.Equals(t, uint8(20), val) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh1, 10) + bs1.Set(bh2, 20) + bs1.Set(bh3, 40) + bs2.Set(bh2, 50) + result = bs1.Sub(bs2) + test.Equals(t, result.Len(), 2) + test.Assert(t, result.Has(bh1) && result.Has(bh3) && !result.Has(bh2), "only bh1 and bh3 should be in result") + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh1, 60) + bs2.Set(bh1, 70) + bs2.Set(bh2, 80) + test.Equals(t, bs1.Sub(bs2).Len(), 0) + }) +} diff --git a/internal/repository/index/master_index.go b/internal/repository/index/master_index.go index 62ccc4f71..f410ebf61 100644 --- a/internal/repository/index/master_index.go +++ b/internal/repository/index/master_index.go @@ -16,13 +16,13 @@ import ( // MasterIndex is a collection of indexes and IDs of chunks that are in the process of being saved. type MasterIndex struct { idx []*Index - pendingBlobs restic.BlobSet + pendingBlobs map[restic.BlobHandle]uint idxMutex sync.RWMutex } // NewMasterIndex creates a new master index. func NewMasterIndex() *MasterIndex { - mi := &MasterIndex{pendingBlobs: restic.NewBlobSet()} + mi := &MasterIndex{pendingBlobs: make(map[restic.BlobHandle]uint)} mi.clear() return mi } @@ -46,10 +46,16 @@ func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) { } // LookupSize queries all known Indexes for the ID and returns the first match. +// Also returns true if the ID is pending. func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() + // also return true if blob is pending + if size, ok := mi.pendingBlobs[bh]; ok { + return size, true + } + for _, idx := range mi.idx { if size, found := idx.LookupSize(bh); found { return size, found @@ -63,13 +69,13 @@ func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) { // Before doing so it checks if this blob is already known. // Returns true if adding was successful and false if the blob // was already known -func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool { +func (mi *MasterIndex) AddPending(bh restic.BlobHandle, size uint) bool { mi.idxMutex.Lock() defer mi.idxMutex.Unlock() // Check if blob is pending or in index - if mi.pendingBlobs.Has(bh) { + if _, ok := mi.pendingBlobs[bh]; ok { return false } @@ -80,30 +86,10 @@ func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool { } // really not known -> insert - mi.pendingBlobs.Insert(bh) + mi.pendingBlobs[bh] = size return true } -// Has queries all known Indexes for the ID and returns the first match. -// Also returns true if the ID is pending. -func (mi *MasterIndex) Has(bh restic.BlobHandle) bool { - mi.idxMutex.RLock() - defer mi.idxMutex.RUnlock() - - // also return true if blob is pending - if mi.pendingBlobs.Has(bh) { - return true - } - - for _, idx := range mi.idx { - if idx.Has(bh) { - return true - } - } - - return false -} - // IDs returns the IDs of all indexes contained in the index. func (mi *MasterIndex) IDs() restic.IDSet { mi.idxMutex.RLock() @@ -165,7 +151,7 @@ func (mi *MasterIndex) storePack(id restic.ID, blobs []restic.Blob) { // delete blobs from pending for _, blob := range blobs { - mi.pendingBlobs.Delete(restic.BlobHandle{Type: blob.Type, ID: blob.ID}) + delete(mi.pendingBlobs, restic.BlobHandle{Type: blob.Type, ID: blob.ID}) } for _, idx := range mi.idx { diff --git a/internal/repository/index/master_index_test.go b/internal/repository/index/master_index_test.go index edf2067b9..39837f53e 100644 --- a/internal/repository/index/master_index_test.go +++ b/internal/repository/index/master_index_test.go @@ -74,9 +74,6 @@ func TestMasterIndex(t *testing.T) { mIdx.Insert(idx2) // test idInIdx1 - found := mIdx.Has(bhInIdx1) - rtest.Equals(t, true, found) - blobs := mIdx.Lookup(bhInIdx1) rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) @@ -85,9 +82,6 @@ func TestMasterIndex(t *testing.T) { rtest.Equals(t, uint(10), size) // test idInIdx2 - found = mIdx.Has(bhInIdx2) - rtest.Equals(t, true, found) - blobs = mIdx.Lookup(bhInIdx2) rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) @@ -96,9 +90,6 @@ func TestMasterIndex(t *testing.T) { rtest.Equals(t, uint(200), size) // test idInIdx12 - found = mIdx.Has(bhInIdx12) - rtest.Equals(t, true, found) - blobs = mIdx.Lookup(bhInIdx12) rtest.Equals(t, 2, len(blobs)) @@ -121,14 +112,91 @@ func TestMasterIndex(t *testing.T) { rtest.Equals(t, uint(80), size) // test not in index - found = mIdx.Has(restic.BlobHandle{ID: restic.NewRandomID(), Type: restic.TreeBlob}) - rtest.Assert(t, !found, "Expected no blobs when fetching with a random id") blobs = mIdx.Lookup(restic.NewRandomBlobHandle()) rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id") _, found = mIdx.LookupSize(restic.NewRandomBlobHandle()) rtest.Assert(t, !found, "Expected no blobs when fetching with a random id") } +func TestMasterIndexAddPending(t *testing.T) { + mIdx := index.NewMasterIndex() + + // Test AddPending: successfully add a new blob + bhPending := restic.NewRandomBlobHandle() + added := mIdx.AddPending(bhPending, 100) + rtest.Equals(t, true, added) + + // Test AddPending: try to add the same blob again (should return false) + added = mIdx.AddPending(bhPending, 200) + rtest.Equals(t, false, added) + + // Test AddPending: try to add a blob that's already in an index (should return false) + bhInIndex := restic.NewRandomBlobHandle() + idx := index.NewIndex() + idx.StorePack(restic.NewRandomID(), []restic.Blob{{ + BlobHandle: bhInIndex, + Length: uint(crypto.CiphertextLength(50)), + Offset: 0, + UncompressedLength: 50, + }}) + mIdx.Insert(idx) + + added = mIdx.AddPending(bhInIndex, 100) + rtest.Equals(t, false, added) + + // Test LookupSize: returns pending blob size when blob is pending + size, found := mIdx.LookupSize(bhPending) + rtest.Equals(t, true, found) + rtest.Equals(t, uint(100), size) +} + +// noopSaver is a no-op implementation of SaverUnpacked for testing. +type noopSaver struct{} + +func (n *noopSaver) Connections() uint { + return 2 +} + +func (n *noopSaver) SaveUnpacked(_ context.Context, _ restic.FileType, buf []byte) (restic.ID, error) { + return restic.Hash(buf), nil +} + +func TestMasterIndexStorePackRemovesPending(t *testing.T) { + mIdx := index.NewMasterIndex() + + // Add a blob as pending + bhPending := restic.NewRandomBlobHandle() + added := mIdx.AddPending(bhPending, 75) + rtest.Equals(t, true, added) + + // Store the blob in a pack + packID := restic.NewRandomID() + blob := restic.Blob{ + BlobHandle: bhPending, + Length: uint(crypto.CiphertextLength(75)), + Offset: 0, + UncompressedLength: 75, + } + saver := &noopSaver{} + err := mIdx.StorePack(context.Background(), packID, []restic.Blob{blob}, saver) + rtest.OK(t, err) + + // Verify it is still found + size, found := mIdx.LookupSize(bhPending) + rtest.Equals(t, true, found) + rtest.Equals(t, uint(75), size) + + // Verify the blob can be found via Lookup from the index + blobs := mIdx.Lookup(bhPending) + rtest.Assert(t, len(blobs) > 0, "blob should be found in index after StorePack") + rtest.Equals(t, packID, blobs[0].PackID) + rtest.Equals(t, bhPending, blobs[0].BlobHandle) + + // Test that adding the same blob as pending again fails (it's now in index) + added = mIdx.AddPending(bhPending, 100) + rtest.Equals(t, false, added) +} + func TestMasterMergeFinalIndexes(t *testing.T) { bhInIdx1 := restic.NewRandomBlobHandle() bhInIdx2 := restic.NewRandomBlobHandle() @@ -521,7 +589,7 @@ func TestRewriteOversizedIndex(t *testing.T) { // verify that blobs are still in the index for _, blob := range blobs { - found := mi2.Has(blob.BlobHandle) + _, found := mi2.LookupSize(blob.BlobHandle) rtest.Assert(t, found, "blob %v missing after rewrite", blob.ID) } diff --git a/internal/repository/prune.go b/internal/repository/prune.go index 250ab9846..843837617 100644 --- a/internal/repository/prune.go +++ b/internal/repository/prune.go @@ -105,7 +105,7 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsed if repo.Config().Version < 2 && opts.RepackUncompressed { return nil, fmt.Errorf("compression requires at least repository format version 2") } - if opts.SmallPackBytes > uint64(repo.packSize()) { + if opts.SmallPackBytes > uint64(repo.PackSize()) { return nil, fmt.Errorf("repack-smaller-than exceeds repository packsize") } @@ -329,12 +329,12 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo *Repository, var repackSmallCandidates []packInfoWithID repoVersion := repo.Config().Version // only repack very small files by default - targetPackSize := repo.packSize() / 25 + targetPackSize := repo.PackSize() / 25 if opts.SmallPackBytes > 0 { targetPackSize = uint(opts.SmallPackBytes) } else if opts.RepackSmall { // consider files with at least 80% of the target size as large enough - targetPackSize = repo.packSize() / 5 * 4 + targetPackSize = repo.PackSize() / 5 * 4 } // loop over all packs and decide what to do @@ -563,7 +563,9 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er if len(plan.repackPacks) != 0 { printer.P("repacking packs\n") bar := printer.NewCounter("packs repacked") - err := Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar, printer.P) + err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + return CopyBlobs(ctx, repo, repo, uploader, plan.repackPacks, plan.keepBlobs, bar, printer.P) + }) if err != nil { return errors.Fatalf("%s", err) } diff --git a/internal/repository/prune_internal_test.go b/internal/repository/prune_internal_test.go index 49a876884..640ab061b 100644 --- a/internal/repository/prune_internal_test.go +++ b/internal/repository/prune_internal_test.go @@ -47,7 +47,7 @@ func TestPruneMaxUnusedDuplicate(t *testing.T) { {bufs[1], bufs[3]}, {bufs[2], bufs[3]}, } { - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { for _, blob := range blobs { id, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, blob, restic.ID{}, true) keep.Insert(restic.BlobHandle{Type: restic.DataBlob, ID: id}) diff --git a/internal/repository/prune_test.go b/internal/repository/prune_test.go index 744de0b14..a363acd41 100644 --- a/internal/repository/prune_test.go +++ b/internal/repository/prune_test.go @@ -25,7 +25,7 @@ func testPrune(t *testing.T, opts repository.PruneOptions, errOnUnused bool) { createRandomBlobs(t, random, repo, 5, 0.5, true) keep, _ := selectBlobs(t, random, repo, 0.5) - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { // duplicate a few blobs to exercise those code paths for blob := range keep { buf, err := repo.LoadBlob(ctx, blob.Type, blob.ID, nil) @@ -133,7 +133,7 @@ func TestPruneSmall(t *testing.T) { const numBlobsCreated = 55 keep := restic.NewBlobSet() - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { // we need a minum of 11 packfiles, each packfile will be about 5 Mb long for i := 0; i < numBlobsCreated; i++ { buf := make([]byte, blobSize) diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 730325afd..c2eaa8f41 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -21,17 +21,18 @@ type repackBlobSet interface { type LogFunc func(msg string, args ...interface{}) -// Repack takes a list of packs together with a list of blobs contained in +// CopyBlobs takes a list of packs together with a list of blobs contained in // these packs. Each pack is loaded and the blobs listed in keepBlobs is saved // into a new pack. Returned is the list of obsolete packs which can then // be removed. // -// The map keepBlobs is modified by Repack, it is used to keep track of which +// The map keepBlobs is modified by CopyBlobs, it is used to keep track of which // blobs have been processed. -func Repack( +func CopyBlobs( ctx context.Context, repo restic.Repository, dstRepo restic.Repository, + dstUploader restic.BlobSaverWithAsync, packs restic.IDSet, keepBlobs repackBlobSet, p *progress.Counter, @@ -49,16 +50,14 @@ func Repack( return errors.New("repack step requires a backend connection limit of at least two") } - return dstRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { - return repack(ctx, repo, dstRepo, uploader, packs, keepBlobs, p, logf) - }) + return repack(ctx, repo, dstRepo, dstUploader, packs, keepBlobs, p, logf) } func repack( ctx context.Context, repo restic.Repository, dstRepo restic.Repository, - uploader restic.BlobSaver, + uploader restic.BlobSaverWithAsync, packs restic.IDSet, keepBlobs repackBlobSet, p *progress.Counter, diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 4d285681f..0c1095301 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -20,7 +20,7 @@ func randomSize(random *rand.Rand, min, max int) int { func createRandomBlobs(t testing.TB, random *rand.Rand, repo restic.Repository, blobs int, pData float32, smallBlobs bool) { // two loops to allow creating multiple pack files for blobs > 0 { - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { for blobs > 0 { blobs-- var ( @@ -70,7 +70,7 @@ func createRandomWrongBlob(t testing.TB, random *rand.Rand, repo restic.Reposito // invert first data byte buf[0] ^= 0xff - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { _, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, buf, id, false) return err })) @@ -150,7 +150,9 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe } func repack(t *testing.T, repo restic.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) { - rtest.OK(t, repository.Repack(context.TODO(), repo, repo, packs, blobs, nil, nil)) + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + return repository.CopyBlobs(ctx, repo, repo, uploader, packs, blobs, nil, nil) + })) for id := range packs { rtest.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})) @@ -263,7 +265,9 @@ func testRepackCopy(t *testing.T, version uint) { _, keepBlobs := selectBlobs(t, random, repo, 0.2) copyPacks := findPacksForBlobs(t, repo, keepBlobs) - rtest.OK(t, repository.Repack(context.TODO(), repoWrapped, dstRepoWrapped, copyPacks, keepBlobs, nil, nil)) + rtest.OK(t, repoWrapped.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + return repository.CopyBlobs(ctx, repoWrapped, dstRepoWrapped, uploader, copyPacks, keepBlobs, nil, nil) + })) rebuildAndReloadIndex(t, dstRepo) for h := range keepBlobs { @@ -299,7 +303,9 @@ func testRepackWrongBlob(t *testing.T, version uint) { _, keepBlobs := selectBlobs(t, random, repo, 0) rewritePacks := findPacksForBlobs(t, repo, keepBlobs) - err := repository.Repack(context.TODO(), repo, repo, rewritePacks, keepBlobs, nil, nil) + err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, nil, nil) + }) if err == nil { t.Fatal("expected repack to fail but got no error") } @@ -330,7 +336,7 @@ func testRepackBlobFallback(t *testing.T, version uint) { modbuf[0] ^= 0xff // create pack with broken copy - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { _, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, modbuf, id, false) return err })) @@ -340,13 +346,15 @@ func testRepackBlobFallback(t *testing.T, version uint) { rewritePacks := findPacksForBlobs(t, repo, keepBlobs) // create pack with valid copy - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { _, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, buf, id, true) return err })) // repack must fallback to valid copy - rtest.OK(t, repository.Repack(context.TODO(), repo, repo, rewritePacks, keepBlobs, nil, nil)) + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, nil, nil) + })) keepBlobs = restic.NewBlobSet(restic.BlobHandle{Type: restic.DataBlob, ID: id}) packs := findPacksForBlobs(t, repo, keepBlobs) diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index a6f4a52b8..0c9d3a43f 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -15,7 +15,7 @@ func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printe bar.SetMax(uint64(len(ids))) defer bar.Done() - err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { // examine all data the indexes have for the pack file for b := range repo.ListPacksFromIndex(ctx, ids) { blobs := b.Blobs diff --git a/internal/repository/repository.go b/internal/repository/repository.go index bb9c6c3ba..e7a1b8c17 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -42,6 +42,8 @@ type Repository struct { opts Options packerWg *errgroup.Group + mainWg *errgroup.Group + blobSaver *sync.WaitGroup uploader *packerUploader treePM *packerManager dataPM *packerManager @@ -154,8 +156,8 @@ func (r *Repository) Config() restic.Config { return r.cfg } -// packSize return the target size of a pack file when uploading -func (r *Repository) packSize() uint { +// PackSize return the target size of a pack file when uploading +func (r *Repository) PackSize() uint { return r.opts.PackSize } @@ -559,11 +561,29 @@ func (r *Repository) removeUnpacked(ctx context.Context, t restic.FileType, id r return r.be.Remove(ctx, backend.Handle{Type: t, Name: id.String()}) } -func (r *Repository) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaver) error) error { +func (r *Repository) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaverWithAsync) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() wg, ctx := errgroup.WithContext(ctx) + // pack uploader + wg.Go below + blob saver (CPU bound) + wg.SetLimit(2 + runtime.GOMAXPROCS(0)) + r.mainWg = wg r.startPackUploader(ctx, wg) + // blob saver are spawned on demand, use wait group to keep track of them + r.blobSaver = &sync.WaitGroup{} wg.Go(func() error { - if err := fn(ctx, &blobSaverRepo{repo: r}); err != nil { + inCallback := true + defer func() { + // when the defer is called while inCallback is true, this means + // that runtime.Goexit was called within `fn`. This should only happen + // if a test uses t.Fatal within `fn`. + if inCallback { + cancel() + } + }() + err := fn(ctx, &blobSaverRepo{repo: r}) + inCallback = false + if err != nil { return err } if err := r.flush(ctx); err != nil { @@ -574,14 +594,6 @@ func (r *Repository) WithBlobUploader(ctx context.Context, fn func(ctx context.C return wg.Wait() } -type blobSaverRepo struct { - repo *Repository -} - -func (r *blobSaverRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) { - return r.repo.saveBlob(ctx, t, buf, id, storeDuplicate) -} - func (r *Repository) startPackUploader(ctx context.Context, wg *errgroup.Group) { if r.packerWg != nil { panic("uploader already started") @@ -590,25 +602,48 @@ func (r *Repository) startPackUploader(ctx context.Context, wg *errgroup.Group) innerWg, ctx := errgroup.WithContext(ctx) r.packerWg = innerWg r.uploader = newPackerUploader(ctx, innerWg, r, r.Connections()) - r.treePM = newPackerManager(r.key, restic.TreeBlob, r.packSize(), r.packerCount, r.uploader.QueuePacker) - r.dataPM = newPackerManager(r.key, restic.DataBlob, r.packSize(), r.packerCount, r.uploader.QueuePacker) + r.treePM = newPackerManager(r.key, restic.TreeBlob, r.PackSize(), r.packerCount, r.uploader.QueuePacker) + r.dataPM = newPackerManager(r.key, restic.DataBlob, r.PackSize(), r.packerCount, r.uploader.QueuePacker) wg.Go(func() error { return innerWg.Wait() }) } +type blobSaverRepo struct { + repo *Repository +} + +func (r *blobSaverRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) { + return r.repo.saveBlob(ctx, t, buf, id, storeDuplicate) +} + +func (r *blobSaverRepo) SaveBlobAsync(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool, cb func(newID restic.ID, known bool, size int, err error)) { + r.repo.saveBlobAsync(ctx, t, buf, id, storeDuplicate, cb) +} + // Flush saves all remaining packs and the index func (r *Repository) flush(ctx context.Context) error { - if err := r.flushPacks(ctx); err != nil { + r.flushBlobSaver() + r.mainWg = nil + + if err := r.flushPackUploader(ctx); err != nil { return err } return r.idx.Flush(ctx, &internalRepository{r}) } +func (r *Repository) flushBlobSaver() { + if r.blobSaver == nil { + return + } + r.blobSaver.Wait() + r.blobSaver = nil +} + // FlushPacks saves all remaining packs. -func (r *Repository) flushPacks(ctx context.Context) error { +func (r *Repository) flushPackUploader(ctx context.Context) error { if r.packerWg == nil { return nil } @@ -640,7 +675,7 @@ func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.Pack return r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) } -// LookupBlobSize returns the size of blob id. +// LookupBlobSize returns the size of blob id. Also returns pending blobs. func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bool) { return r.idx.LookupSize(restic.BlobHandle{Type: tpe, ID: id}) } @@ -787,6 +822,22 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest return invalid, nil } +func (r *Repository) NewAssociatedBlobSet() restic.AssociatedBlobSet { + return &associatedBlobSet{*index.NewAssociatedSet[struct{}](r.idx)} +} + +// associatedBlobSet is a wrapper around index.AssociatedSet to implement the restic.AssociatedBlobSet interface. +type associatedBlobSet struct { + index.AssociatedSet[struct{}] +} + +func (s *associatedBlobSet) Intersect(other restic.AssociatedBlobSet) restic.AssociatedBlobSet { + return &associatedBlobSet{*s.AssociatedSet.Intersect(other)} +} +func (s *associatedBlobSet) Sub(other restic.AssociatedBlobSet) restic.AssociatedBlobSet { + return &associatedBlobSet{*s.AssociatedSet.Sub(other)} +} + // prepareCache initializes the local cache. indexIDs is the list of IDs of // index files still present in the repo. func (r *Repository) prepareCache() error { @@ -968,7 +1019,7 @@ func (r *Repository) saveBlob(ctx context.Context, t restic.BlobType, buf []byte } // first try to add to pending blobs; if not successful, this blob is already known - known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t}) + known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t}, uint(len(buf))) // only save when needed or explicitly told if !known || storeDuplicate { @@ -978,6 +1029,19 @@ func (r *Repository) saveBlob(ctx context.Context, t restic.BlobType, buf []byte return newID, known, size, err } +func (r *Repository) saveBlobAsync(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool, cb func(newID restic.ID, known bool, size int, err error)) { + r.mainWg.Go(func() error { + if ctx.Err() != nil { + // fail fast if the context is cancelled + cb(restic.ID{}, false, 0, ctx.Err()) + return ctx.Err() + } + newID, known, size, err := r.saveBlob(ctx, t, buf, id, storeDuplicate) + cb(newID, known, size, err) + return err + }) +} + type backendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error type loadBlobFn func(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) ([]byte, error) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 2a181312c..f2ef1d082 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -4,11 +4,13 @@ import ( "bytes" "context" "crypto/sha256" + "fmt" "io" "math/rand" "path/filepath" "strings" "sync" + "sync/atomic" "testing" "time" @@ -51,7 +53,7 @@ func testSave(t *testing.T, version uint, calculateID bool) { id := restic.Hash(data) - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { // save inputID := restic.ID{} if !calculateID { @@ -97,7 +99,7 @@ func testSavePackMerging(t *testing.T, targetPercentage int, expectedPacks int) }) var ids restic.IDs - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { // add blobs with size targetPercentage / 100 * repo.PackSize to the repository blobSize := repository.MinPackSize / 100 for range targetPercentage { @@ -147,7 +149,7 @@ func benchmarkSaveAndEncrypt(t *testing.B, version uint) { t.ResetTimer() t.SetBytes(int64(size)) - _ = repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + _ = repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { for i := 0; i < t.N; i++ { _, _, _, err = uploader.SaveBlob(ctx, restic.DataBlob, data, id, true) rtest.OK(t, err) @@ -168,7 +170,7 @@ func testLoadBlob(t *testing.T, version uint) { rtest.OK(t, err) var id restic.ID - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { var err error id, _, _, err = uploader.SaveBlob(ctx, restic.DataBlob, buf, restic.ID{}, false) return err @@ -196,7 +198,7 @@ func TestLoadBlobBroken(t *testing.T) { buf := rtest.Random(42, 1000) var id restic.ID - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { var err error id, _, _, err = uploader.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) return err @@ -225,7 +227,7 @@ func benchmarkLoadBlob(b *testing.B, version uint) { rtest.OK(b, err) var id restic.ID - rtest.OK(b, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(b, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { var err error id, _, _, err = uploader.SaveBlob(ctx, restic.DataBlob, buf, restic.ID{}, false) return err @@ -361,7 +363,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) { // saveRandomDataBlobs generates random data blobs and saves them to the repository. func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) { - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { for i := 0; i < num; i++ { size := rand.Int() % sizeMax @@ -432,7 +434,7 @@ func TestListPack(t *testing.T) { buf := rtest.Random(42, 1000) var id restic.ID - rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { var err error id, _, _, err = uploader.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) return err @@ -487,3 +489,65 @@ func TestNoDoubleInit(t *testing.T) { err = repo.Init(context.TODO(), r.Config().Version, rtest.TestPassword, &pol) rtest.Assert(t, strings.Contains(err.Error(), "repository already contains snapshots"), "expected already contains snapshots error, got %q", err) } + +func TestSaveBlobAsync(t *testing.T) { + repo, _, _ := repository.TestRepositoryWithVersion(t, 2) + ctx := context.Background() + + type result struct { + id restic.ID + known bool + size int + err error + } + numCalls := 10 + results := make([]result, numCalls) + var resultsMutex sync.Mutex + + err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + var wg sync.WaitGroup + wg.Add(numCalls) + for i := 0; i < numCalls; i++ { + // Use unique data for each call + testData := []byte(fmt.Sprintf("test blob data %d", i)) + uploader.SaveBlobAsync(ctx, restic.DataBlob, testData, restic.ID{}, false, + func(newID restic.ID, known bool, size int, err error) { + defer wg.Done() + resultsMutex.Lock() + results[i] = result{newID, known, size, err} + resultsMutex.Unlock() + }) + } + wg.Wait() + return nil + }) + rtest.OK(t, err) + + for i, result := range results { + testData := []byte(fmt.Sprintf("test blob data %d", i)) + expectedID := restic.Hash(testData) + rtest.Assert(t, result.err == nil, "result %d: unexpected error %v", i, result.err) + rtest.Assert(t, result.id.Equal(expectedID), "result %d: expected ID %v, got %v", i, expectedID, result.id) + rtest.Assert(t, !result.known, "result %d: expected unknown blob", i) + } +} + +func TestSaveBlobAsyncErrorHandling(t *testing.T) { + repo, _, _ := repository.TestRepositoryWithVersion(t, 2) + ctx, cancel := context.WithCancel(context.Background()) + + var callbackCalled atomic.Bool + + err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + cancel() + // Callback must be called even if the context is canceled + uploader.SaveBlobAsync(ctx, restic.DataBlob, []byte("test blob data"), restic.ID{}, false, + func(newID restic.ID, known bool, size int, err error) { + callbackCalled.Store(true) + }) + return nil + }) + + rtest.Assert(t, errors.Is(err, context.Canceled), "expected context canceled error, got %v", err) + rtest.Assert(t, callbackCalled.Load(), "callback was not called") +} diff --git a/internal/restic/lock_unix.go b/internal/restic/lock_unix.go index 393dab5ba..ab54daf8e 100644 --- a/internal/restic/lock_unix.go +++ b/internal/restic/lock_unix.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package restic diff --git a/internal/restic/repository.go b/internal/restic/repository.go index cf3ec7834..c7f326823 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -2,6 +2,7 @@ package restic import ( "context" + "iter" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" @@ -18,6 +19,7 @@ type Repository interface { // Connections returns the maximum number of concurrent backend operations Connections() uint Config() Config + PackSize() uint Key() *crypto.Key LoadIndex(ctx context.Context, p TerminalCounterFactory) error @@ -25,6 +27,7 @@ type Repository interface { LookupBlob(t BlobType, id ID) []PackedBlob LookupBlobSize(t BlobType, id ID) (size uint, exists bool) + NewAssociatedBlobSet() AssociatedBlobSet // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. ListBlobs(ctx context.Context, fn func(PackedBlob)) error @@ -39,7 +42,7 @@ type Repository interface { // WithUploader starts the necessary workers to upload new blobs. Once the callback returns, // the workers are stopped and the index is written to the repository. The callback must use // the passed context and must not keep references to any of its parameters after returning. - WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader BlobSaver) error) error + WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader BlobSaverWithAsync) error) error // List calls the function fn for each file of type t in the repository. // When an error is returned by fn, processing stops and List() returns the @@ -159,11 +162,23 @@ type BlobLoader interface { } type WithBlobUploader interface { - WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader BlobSaver) error) error + WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader BlobSaverWithAsync) error) error +} + +type BlobSaverWithAsync interface { + BlobSaver + BlobSaverAsync } type BlobSaver interface { - SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) + // SaveBlob saves a blob to the repository. ctx must be derived from the context created by WithBlobUploader. + SaveBlob(ctx context.Context, tpe BlobType, buf []byte, id ID, storeDuplicate bool) (newID ID, known bool, sizeInRepo int, err error) +} + +type BlobSaverAsync interface { + // SaveBlobAsync saves a blob to the repository. ctx must be derived from the context created by WithBlobUploader. + // The callback is called asynchronously from a different goroutine. + SaveBlobAsync(ctx context.Context, tpe BlobType, buf []byte, id ID, storeDuplicate bool, cb func(newID ID, known bool, sizeInRepo int, err error)) } // Loader loads a blob from a repository. @@ -185,3 +200,13 @@ type FindBlobSet interface { Has(bh BlobHandle) bool Insert(bh BlobHandle) } + +type AssociatedBlobSet interface { + Has(bh BlobHandle) bool + Insert(bh BlobHandle) + Delete(bh BlobHandle) + Len() int + Keys() iter.Seq[BlobHandle] + Intersect(other AssociatedBlobSet) AssociatedBlobSet + Sub(other AssociatedBlobSet) AssociatedBlobSet +} diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 166bf1ff8..d197bcf5b 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -217,6 +217,8 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { wg, ctx := errgroup.WithContext(ctx) downloadCh := make(chan *packInfo) + // close all files when finished + defer r.filesWriter.flush() worker := func() error { for pack := range downloadCh { if err := r.downloadPack(ctx, pack); err != nil { diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index d6f78f2d7..2b60f7185 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -7,6 +7,7 @@ import ( "syscall" "github.com/cespare/xxhash/v2" + "github.com/hashicorp/golang-lru/v2/simplelru" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -20,6 +21,8 @@ import ( type filesWriter struct { buckets []filesWriterBucket allowRecursiveDelete bool + cacheMu sync.Mutex + cache *simplelru.LRU[string, *partialFile] } type filesWriterBucket struct { @@ -34,13 +37,27 @@ type partialFile struct { } func newFilesWriter(count int, allowRecursiveDelete bool) *filesWriter { - buckets := make([]filesWriterBucket, count) - for b := 0; b < count; b++ { + // use a large number of buckets to minimize bucket contention + // creating a new file can be slow, so make sure that files typically end up in different buckets. + buckets := make([]filesWriterBucket, 1024) + for b := 0; b < len(buckets); b++ { buckets[b].files = make(map[string]*partialFile) } + + cache, err := simplelru.NewLRU[string, *partialFile](count+50, func(_ string, wr *partialFile) { + // close the file only when it is not in use + if wr.users == 0 { + _ = wr.Close() + } + }) + if err != nil { + panic(err) // can't happen + } + return &filesWriter{ buckets: buckets, allowRecursiveDelete: allowRecursiveDelete, + cache: cache, } } @@ -173,6 +190,24 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create bucket.files[path].users++ return wr, nil } + + // Check the global LRU cache for a cached file handle + w.cacheMu.Lock() + cached, ok := w.cache.Get(path) + if ok { + // mark as in use to prevent closing on remove call below + cached.users++ + + w.cache.Remove(path) + w.cacheMu.Unlock() + + // Use the cached file handle + bucket.files[path] = cached + return cached, nil + } + w.cacheMu.Unlock() + + // Not in cache, open/create the file var f *os.File var err error if createSize >= 0 { @@ -194,11 +229,14 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create bucket.lock.Lock() defer bucket.lock.Unlock() - if bucket.files[path].users == 1 { - delete(bucket.files, path) - return wr.Close() - } bucket.files[path].users-- + if bucket.files[path].users == 0 { + delete(bucket.files, path) + // Add to cache to allow re-use. Cache will close files on overflow. + w.cacheMu.Lock() + w.cache.Add(path, wr) + w.cacheMu.Unlock() + } return nil } @@ -217,3 +255,10 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create return releaseWriter(wr) } + +func (w *filesWriter) flush() { + w.cacheMu.Lock() + defer w.cacheMu.Unlock() + + w.cache.Purge() +} diff --git a/internal/restorer/fileswriter_other_test.go b/internal/restorer/fileswriter_other_test.go index 530a190e5..756cb5bac 100644 --- a/internal/restorer/fileswriter_other_test.go +++ b/internal/restorer/fileswriter_other_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package restorer diff --git a/internal/restorer/fileswriter_test.go b/internal/restorer/fileswriter_test.go index 9ea8767b8..dfdccb647 100644 --- a/internal/restorer/fileswriter_test.go +++ b/internal/restorer/fileswriter_test.go @@ -30,6 +30,8 @@ func TestFilesWriterBasic(t *testing.T) { rtest.OK(t, w.writeToFile(f2, []byte{2}, 1, -1, false)) rtest.Equals(t, 0, len(w.buckets[0].files)) + w.flush() + buf, err := os.ReadFile(f1) rtest.OK(t, err) rtest.Equals(t, []byte{1, 1}, buf) @@ -51,11 +53,13 @@ func TestFilesWriterRecursiveOverwrite(t *testing.T) { err := w.writeToFile(path, []byte{1}, 0, 2, false) rtest.Assert(t, errors.Is(err, notEmptyDirError()), "unexpected error got %v", err) rtest.Equals(t, 0, len(w.buckets[0].files)) + w.flush() // must replace directory w = newFilesWriter(1, true) rtest.OK(t, w.writeToFile(path, []byte{1, 1}, 0, 2, false)) rtest.Equals(t, 0, len(w.buckets[0].files)) + w.flush() buf, err := os.ReadFile(path) rtest.OK(t, err) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 22ab196a5..8454591e4 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -163,15 +163,18 @@ func (res *Restorer) traverseTreeInner(ctx context.Context, target, location str } if res.opts.Delete { - filenames = make([]string, 0, len(tree.Nodes)) + filenames = make([]string, 0) } - for i, node := range tree.Nodes { + for item := range tree { + if item.Error != nil { + debug.Log("error iterating tree %v: %v", treeID, item.Error) + return nil, hasRestored, res.sanitizeError(location, item.Error) + } + node := item.Node if ctx.Err() != nil { return nil, hasRestored, ctx.Err() } - // allow GC of tree node - tree.Nodes[i] = nil if res.opts.Delete { // just track all files included in the tree node to simplify the control flow. // tracking too many files does not matter except for a slightly elevated memory usage diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index fe9db6b33..337a99918 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -79,7 +79,7 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tree := &data.Tree{} + tree := make([]*data.Node, 0, len(nodes)) for name, n := range nodes { inode++ switch node := n.(type) { @@ -107,7 +107,7 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u if mode == 0 { mode = 0644 } - err := tree.Insert(&data.Node{ + tree = append(tree, &data.Node{ Type: data.NodeTypeFile, Mode: mode, ModTime: node.ModTime, @@ -120,9 +120,8 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u Links: lc, GenericAttributes: getGenericAttributes(node.attributes, false), }) - rtest.OK(t, err) case Symlink: - err := tree.Insert(&data.Node{ + tree = append(tree, &data.Node{ Type: data.NodeTypeSymlink, Mode: os.ModeSymlink | 0o777, ModTime: node.ModTime, @@ -133,7 +132,6 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u Inode: inode, Links: 1, }) - rtest.OK(t, err) case Dir: id := saveDir(t, repo, node.Nodes, inode, getGenericAttributes) @@ -142,7 +140,7 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u mode = 0755 } - err := tree.Insert(&data.Node{ + tree = append(tree, &data.Node{ Type: data.NodeTypeDir, Mode: mode, ModTime: node.ModTime, @@ -152,18 +150,12 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u Subtree: &id, GenericAttributes: getGenericAttributes(node.attributes, false), }) - rtest.OK(t, err) default: t.Fatalf("unknown node type %T", node) } } - id, err := data.SaveTree(ctx, repo, tree) - if err != nil { - t.Fatal(err) - } - - return id + return data.TestSaveNodes(t, ctx, repo, tree) } func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[data.GenericAttributeType]json.RawMessage)) (*data.Snapshot, restic.ID) { @@ -171,7 +163,7 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGe defer cancel() var treeID restic.ID - rtest.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error { + rtest.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { treeID = saveDir(t, uploader, snapshot.Nodes, 1000, getGenericAttributes) return nil })) diff --git a/internal/restorer/restorer_unix.go b/internal/restorer/restorer_unix.go index 7316f7b5d..b013564f1 100644 --- a/internal/restorer/restorer_unix.go +++ b/internal/restorer/restorer_unix.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package restorer diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index c4e8149b2..a7912ea03 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package restorer diff --git a/internal/restorer/restorer_windows.go b/internal/restorer/restorer_windows.go index 9ddc0a932..2be86c3a4 100644 --- a/internal/restorer/restorer_windows.go +++ b/internal/restorer/restorer_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package restorer diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go index 348f02240..9bc680c51 100644 --- a/internal/restorer/restorer_windows_test.go +++ b/internal/restorer/restorer_windows_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package restorer diff --git a/internal/restorer/truncate_other.go b/internal/restorer/truncate_other.go index ed7ab04c5..25bdb52b3 100644 --- a/internal/restorer/truncate_other.go +++ b/internal/restorer/truncate_other.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package restorer diff --git a/internal/selfupdate/download_unix.go b/internal/selfupdate/download_unix.go index bc1762948..883733cd5 100644 --- a/internal/selfupdate/download_unix.go +++ b/internal/selfupdate/download_unix.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package selfupdate diff --git a/internal/selfupdate/download_windows.go b/internal/selfupdate/download_windows.go index 50480eab6..3e771cfd0 100644 --- a/internal/selfupdate/download_windows.go +++ b/internal/selfupdate/download_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package selfupdate diff --git a/internal/terminal/foreground_test.go b/internal/terminal/foreground_test.go index c8392ab88..2e711cc1a 100644 --- a/internal/terminal/foreground_test.go +++ b/internal/terminal/foreground_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package terminal_test diff --git a/internal/terminal/terminal_unix.go b/internal/terminal/terminal_unix.go index 732219bb4..2065f50cf 100644 --- a/internal/terminal/terminal_unix.go +++ b/internal/terminal/terminal_unix.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package terminal diff --git a/internal/terminal/terminal_windows.go b/internal/terminal/terminal_windows.go index fffabc5ee..163173286 100644 --- a/internal/terminal/terminal_windows.go +++ b/internal/terminal/terminal_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package terminal diff --git a/internal/test/helpers.go b/internal/test/helpers.go index 3387d36df..e3fded66e 100644 --- a/internal/test/helpers.go +++ b/internal/test/helpers.go @@ -48,7 +48,7 @@ func OKs(tb testing.TB, errs []error) { // Equals fails the test if exp is not equal to act. // msg is optional message to be printed, first param being format string and rest being arguments. -func Equals(tb testing.TB, exp, act interface{}, msgs ...string) { +func Equals[T any](tb testing.TB, exp, act T, msgs ...string) { tb.Helper() if !reflect.DeepEqual(exp, act) { var msgString string diff --git a/internal/ui/signals/signals_bsd.go b/internal/ui/signals/signals_bsd.go index d96e48c4e..d3318f5b0 100644 --- a/internal/ui/signals/signals_bsd.go +++ b/internal/ui/signals/signals_bsd.go @@ -1,5 +1,4 @@ //go:build darwin || dragonfly || freebsd || netbsd || openbsd -// +build darwin dragonfly freebsd netbsd openbsd package signals diff --git a/internal/ui/signals/signals_sysv.go b/internal/ui/signals/signals_sysv.go index 9480c1c99..e6472838f 100644 --- a/internal/ui/signals/signals_sysv.go +++ b/internal/ui/signals/signals_sysv.go @@ -1,5 +1,4 @@ //go:build aix || linux || solaris -// +build aix linux solaris package signals diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index 3265c7a04..94ea6adaa 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "slices" "strings" "sync" @@ -204,6 +205,7 @@ func (t *Terminal) Run(ctx context.Context) { // run listens on the channels and updates the terminal screen. func (t *Terminal) run(ctx context.Context) { var status []string + var lastWrittenStatus []string for { select { case <-ctx.Done(): @@ -240,6 +242,7 @@ func (t *Terminal) run(ctx context.Context) { } t.writeStatus(status) + lastWrittenStatus = append([]string{}, status...) case stat := <-t.status: status = append(status[:0], stat.lines...) @@ -248,7 +251,11 @@ func (t *Terminal) run(ctx context.Context) { continue } - t.writeStatus(status) + if !slices.Equal(status, lastWrittenStatus) { + t.writeStatus(status) + // Copy the status slice to avoid aliasing + lastWrittenStatus = append([]string{}, status...) + } } } } @@ -287,6 +294,7 @@ func (t *Terminal) writeStatus(status []string) { // runWithoutStatus listens on the channels and just prints out the messages, // without status lines. func (t *Terminal) runWithoutStatus(ctx context.Context) { + var lastStatus []string for { select { case <-ctx.Done(): @@ -309,11 +317,15 @@ func (t *Terminal) runWithoutStatus(ctx context.Context) { } case stat := <-t.status: - for _, line := range stat.lines { - // Ensure that each message ends with exactly one newline. - if _, err := fmt.Fprintln(t.wr, strings.TrimRight(line, "\n")); err != nil { - _, _ = fmt.Fprintf(t.errWriter, "write failed: %v\n", err) + if !slices.Equal(stat.lines, lastStatus) { + for _, line := range stat.lines { + // Ensure that each message ends with exactly one newline. + if _, err := fmt.Fprintln(t.wr, strings.TrimRight(line, "\n")); err != nil { + _, _ = fmt.Fprintf(t.errWriter, "write failed: %v\n", err) + } } + // Copy the status slice to avoid aliasing + lastStatus = append([]string{}, stat.lines...) } } } diff --git a/internal/ui/termstatus/status_test.go b/internal/ui/termstatus/status_test.go index f65bb096f..b19e00557 100644 --- a/internal/ui/termstatus/status_test.go +++ b/internal/ui/termstatus/status_test.go @@ -128,7 +128,7 @@ func TestRawInputOutput(t *testing.T) { defer cancel() rtest.Equals(t, input, term.InputRaw()) rtest.Equals(t, false, term.InputIsTerminal()) - rtest.Equals(t, &output, term.OutputRaw()) + rtest.Equals(t, io.Writer(&output), term.OutputRaw()) rtest.Equals(t, false, term.OutputIsTerminal()) rtest.Equals(t, false, term.CanUpdateStatus()) } diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index bd05b90d7..f54b34c47 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -11,8 +11,9 @@ import ( ) type NodeRewriteFunc func(node *data.Node, path string) *data.Node -type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (*data.Tree, error) +type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (data.TreeNodeIterator, error) type QueryRewrittenSizeFunc func() SnapshotSize +type NodeKeepEmptyDirectoryFunc func(path string) bool type SnapshotSize struct { FileCount uint @@ -21,7 +22,8 @@ type SnapshotSize struct { type RewriteOpts struct { // return nil to remove the node - RewriteNode NodeRewriteFunc + RewriteNode NodeRewriteFunc + KeepEmptyDirectory NodeKeepEmptyDirectoryFunc // decide what to do with a tree that could not be loaded. Return nil to remove the node. By default the load error is returned which causes the operation to fail. RewriteFailedTree FailedTreeRewriteFunc @@ -52,14 +54,19 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { } if rw.opts.RewriteFailedTree == nil { // fail with error by default - rw.opts.RewriteFailedTree = func(_ restic.ID, _ string, err error) (*data.Tree, error) { + rw.opts.RewriteFailedTree = func(_ restic.ID, _ string, err error) (data.TreeNodeIterator, error) { return nil, err } } + if rw.opts.KeepEmptyDirectory == nil { + rw.opts.KeepEmptyDirectory = func(_ string) bool { + return true + } + } return rw } -func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc) (*TreeRewriter, QueryRewrittenSizeFunc) { +func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc, keepEmptyDirecoryFilter NodeKeepEmptyDirectoryFunc) (*TreeRewriter, QueryRewrittenSizeFunc) { var count uint var size uint64 @@ -72,7 +79,8 @@ func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc) (*TreeRewriter, QueryR } return node }, - DisableNodeCache: true, + DisableNodeCache: true, + KeepEmptyDirectory: keepEmptyDirecoryFilter, }) ss := func() SnapshotSize { @@ -117,15 +125,26 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, loader restic.BlobLoader if nodeID != testID { return restic.ID{}, fmt.Errorf("cannot encode tree at %q without losing information", nodepath) } + + // reload the tree to get a new iterator + curTree, err = data.LoadTree(ctx, loader, nodeID) + if err != nil { + // shouldn't fail as the first load was successful + return restic.ID{}, fmt.Errorf("failed to reload tree %v: %w", nodeID, err) + } } debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str()) - tb := data.NewTreeJSONBuilder() - for _, node := range curTree.Nodes { + tb := data.NewTreeWriter(saver) + for item := range curTree { if ctx.Err() != nil { return restic.ID{}, ctx.Err() } + if item.Error != nil { + return restic.ID{}, item.Error + } + node := item.Node path := path.Join(nodepath, node.Name) node = t.opts.RewriteNode(node, path) @@ -148,6 +167,8 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, loader restic.BlobLoader newID, err := t.RewriteTree(ctx, loader, saver, path, subtree) if err != nil { return restic.ID{}, err + } else if err == nil && newID.IsNull() { + continue } node.Subtree = &newID err = tb.AddNode(node) @@ -156,13 +177,14 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, loader restic.BlobLoader } } - tree, err := tb.Finalize() + newTreeID, err := tb.Finalize(ctx) if err != nil { return restic.ID{}, err } + if tb.Count() == 0 && !t.opts.KeepEmptyDirectory(nodepath) { + return restic.ID{}, nil + } - // Save new tree - newTreeID, _, _, err := saver.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false) if t.replaces != nil { t.replaces[nodeID] = newTreeID } diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go index 9290a62d5..7cb34c9d0 100644 --- a/internal/walker/rewriter_test.go +++ b/internal/walker/rewriter_test.go @@ -2,42 +2,14 @@ package walker import ( "context" + "slices" "testing" - "github.com/pkg/errors" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) -// WritableTreeMap also support saving -type WritableTreeMap struct { - TreeMap -} - -func (t WritableTreeMap) SaveBlob(_ context.Context, tpe restic.BlobType, buf []byte, id restic.ID, _ bool) (newID restic.ID, known bool, size int, err error) { - if tpe != restic.TreeBlob { - return restic.ID{}, false, 0, errors.New("can only save trees") - } - - if id.IsNull() { - id = restic.Hash(buf) - } - _, ok := t.TreeMap[id] - if ok { - return id, false, 0, nil - } - - t.TreeMap[id] = append([]byte{}, buf...) - return id, true, len(buf), nil -} - -func (t WritableTreeMap) Dump(test testing.TB) { - for k, v := range t.TreeMap { - test.Logf("%v: %v", k, string(v)) - } -} - type checkRewriteFunc func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) // checkRewriteItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'. @@ -279,7 +251,7 @@ func TestRewriter(t *testing.T) { test.newTree = test.tree } expRepo, expRoot := BuildTreeMap(test.newTree) - modrepo := WritableTreeMap{repo} + modrepo := data.TestWritableTreeMap{TestTreeMap: repo} ctx, cancel := context.WithCancel(context.TODO()) defer cancel() @@ -297,7 +269,7 @@ func TestRewriter(t *testing.T) { t.Log("Got") modrepo.Dump(t) t.Log("Expected") - WritableTreeMap{expRepo}.Dump(t) + data.TestWritableTreeMap{TestTreeMap: expRepo}.Dump(t) } }) } @@ -320,7 +292,7 @@ func TestSnapshotSizeQuery(t *testing.T) { t.Run("", func(t *testing.T) { repo, root := BuildTreeMap(tree) expRepo, expRoot := BuildTreeMap(newTree) - modrepo := WritableTreeMap{repo} + modrepo := data.TestWritableTreeMap{TestTreeMap: repo} ctx, cancel := context.WithCancel(context.TODO()) defer cancel() @@ -334,7 +306,7 @@ func TestSnapshotSizeQuery(t *testing.T) { } return node } - rewriter, querySize := NewSnapshotSizeRewriter(rewriteNode) + rewriter, querySize := NewSnapshotSizeRewriter(rewriteNode, nil) newRoot, err := rewriter.RewriteTree(ctx, modrepo, modrepo, "/", root) if err != nil { t.Error(err) @@ -351,17 +323,85 @@ func TestSnapshotSizeQuery(t *testing.T) { t.Log("Got") modrepo.Dump(t) t.Log("Expected") - WritableTreeMap{expRepo}.Dump(t) + data.TestWritableTreeMap{TestTreeMap: expRepo}.Dump(t) } }) } +func TestRewriterKeepEmptyDirectory(t *testing.T) { + var paths []string + tests := []struct { + name string + keepEmpty NodeKeepEmptyDirectoryFunc + assert func(t *testing.T, newRoot restic.ID) + }{ + { + name: "Keep", + keepEmpty: func(string) bool { return true }, + assert: func(t *testing.T, newRoot restic.ID) { + _, expRoot := BuildTreeMap(TestTree{"empty": TestTree{}}) + test.Assert(t, newRoot == expRoot, "expected empty dir kept") + }, + }, + { + name: "Drop subdir only", + keepEmpty: func(p string) bool { return p != "/empty" }, + assert: func(t *testing.T, newRoot restic.ID) { + _, expRoot := BuildTreeMap(TestTree{}) + test.Assert(t, newRoot == expRoot, "expected empty root") + }, + }, + { + name: "Drop all", + keepEmpty: func(string) bool { return false }, + assert: func(t *testing.T, newRoot restic.ID) { + test.Assert(t, newRoot.IsNull(), "expected null root") + }, + }, + { + name: "Paths", + keepEmpty: func(p string) bool { + paths = append(paths, p) + return p != "/empty" + }, + assert: func(t *testing.T, newRoot restic.ID) { + test.Assert(t, len(paths) >= 2, "expected at least two KeepEmptyDirectory calls") + var hasRoot, hasEmpty bool + for _, p := range paths { + if p == "/" { + hasRoot = true + } + if p == "/empty" { + hasEmpty = true + } + } + test.Assert(t, hasRoot && hasEmpty, "expected paths \"/\" and \"/empty\", got %v", paths) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + repo, root := BuildTreeMap(TestTree{"empty": TestTree{}}) + modrepo := data.TestWritableTreeMap{TestTreeMap: repo} + + rw := NewTreeRewriter(RewriteOpts{KeepEmptyDirectory: tc.keepEmpty}) + newRoot, err := rw.RewriteTree(ctx, modrepo, modrepo, "/", root) + test.OK(t, err) + tc.assert(t, newRoot) + }) + } +} + func TestRewriterFailOnUnknownFields(t *testing.T) { - tm := WritableTreeMap{TreeMap{}} + tm := data.TestWritableTreeMap{TestTreeMap: data.TestTreeMap{}} node := []byte(`{"nodes":[{"name":"subfile","type":"file","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","uid":0,"gid":0,"content":null,"unknown_field":42}]}`) id := restic.Hash(node) - tm.TreeMap[id] = node + tm.TestTreeMap[id] = node ctx, cancel := context.WithCancel(context.TODO()) defer cancel() @@ -392,7 +432,7 @@ func TestRewriterFailOnUnknownFields(t *testing.T) { } func TestRewriterTreeLoadError(t *testing.T) { - tm := WritableTreeMap{TreeMap{}} + tm := data.TestWritableTreeMap{TestTreeMap: data.TestTreeMap{}} id := restic.NewRandomID() ctx, cancel := context.WithCancel(context.TODO()) @@ -405,16 +445,15 @@ func TestRewriterTreeLoadError(t *testing.T) { t.Fatal("missing error on unloadable tree") } - replacementTree := &data.Tree{Nodes: []*data.Node{{Name: "replacement", Type: data.NodeTypeFile, Size: 42}}} - replacementID, err := data.SaveTree(ctx, tm, replacementTree) - test.OK(t, err) + replacementNode := &data.Node{Name: "replacement", Type: data.NodeTypeFile, Size: 42} + replacementID := data.TestSaveNodes(t, ctx, tm, []*data.Node{replacementNode}) rewriter = NewTreeRewriter(RewriteOpts{ - RewriteFailedTree: func(nodeID restic.ID, path string, err error) (*data.Tree, error) { + RewriteFailedTree: func(nodeID restic.ID, path string, err error) (data.TreeNodeIterator, error) { if nodeID != id || path != "/" { t.Fail() } - return replacementTree, nil + return slices.Values([]data.NodeOrError{{Node: replacementNode}}), nil }, }) newRoot, err := rewriter.RewriteTree(ctx, tm, tm, "/", id) diff --git a/internal/walker/walker.go b/internal/walker/walker.go index 67c4a9d03..8347c28c4 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -3,7 +3,6 @@ package walker import ( "context" "path" - "sort" "github.com/pkg/errors" @@ -52,15 +51,15 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor W // walk recursively traverses the tree, ignoring subtrees when the ID of the // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID // will be added to ignoreTrees by walk. -func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *data.Tree, visitor WalkVisitor) (err error) { - sort.Slice(tree.Nodes, func(i, j int) bool { - return tree.Nodes[i].Name < tree.Nodes[j].Name - }) - - for _, node := range tree.Nodes { +func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree data.TreeNodeIterator, visitor WalkVisitor) (err error) { + for item := range tree { + if item.Error != nil { + return item.Error + } if ctx.Err() != nil { return ctx.Err() } + node := item.Node p := path.Join(prefix, node.Name) diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index fa561bf19..fad95476d 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -6,7 +6,6 @@ import ( "sort" "testing" - "github.com/pkg/errors" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -20,13 +19,13 @@ type TestFile struct { Size uint64 } -func BuildTreeMap(tree TestTree) (m TreeMap, root restic.ID) { - m = TreeMap{} +func BuildTreeMap(tree TestTree) (m data.TestTreeMap, root restic.ID) { + m = data.TestTreeMap{} id := buildTreeMap(tree, m) return m, id } -func buildTreeMap(tree TestTree, m TreeMap) restic.ID { +func buildTreeMap(tree TestTree, m data.TestTreeMap) restic.ID { tb := data.NewTreeJSONBuilder() var names []string for name := range tree { @@ -75,24 +74,6 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID { return id } -// TreeMap returns the trees from the map on LoadTree. -type TreeMap map[restic.ID][]byte - -func (t TreeMap) LoadBlob(_ context.Context, tpe restic.BlobType, id restic.ID, _ []byte) ([]byte, error) { - if tpe != restic.TreeBlob { - return nil, errors.New("can only load trees") - } - tree, ok := t[id] - if !ok { - return nil, errors.New("tree not found") - } - return tree, nil -} - -func (t TreeMap) Connections() uint { - return 2 -} - // checkFunc returns a function suitable for walking the tree to check // something, and a function which will check the final result. type checkFunc func(t testing.TB) (walker WalkFunc, leaveDir func(path string) error, final func(testing.TB, error))